diff --git a/.env.example b/.env.example index d365a35..de4189c 100644 --- a/.env.example +++ b/.env.example @@ -29,6 +29,14 @@ PUBLIC_IMAGE_BASE_URL=http://localhost:8888 # Gateway proxies /api and Swagger routes to this upstream. GATEWAY_BACKEND_UPSTREAM=http://localhost:3001 +GATEWAY_L1_MAX_ENTRIES=256 +GATEWAY_L1_TTL_MS=600000 + +# MVP dev mode: mock source host allowlist without DB/admin CRUD. +SOURCE_HOST_ALLOW_ALL=false +SOURCE_ALLOWED_HOSTS=storage.yandexcloud.net +IMAGE_ALLOW_CUSTOM_TRANSFORMS=true +IMAGE_ENSURE_WAIT_MS=15000 # Dev imgproxy is exposed only on localhost. IMGPROXY_PORT=18080 diff --git a/README.md b/README.md index a09a4f5..d6f1393 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Image Platform - отдельная площадка для управления ## Статус -Сейчас создан базовый monorepo, dev-инфраструктура, NestJS backend с Swagger, чистый Vite React TS admin, Fastify gateway skeleton, Drizzle database package, shared queue/storage packages и worker skeleton. +Сейчас создан базовый monorepo, dev-инфраструктура, NestJS backend с Swagger, чистый Vite React TS admin, Fastify gateway, Drizzle database package, shared queue/storage packages и worker. Минимальный read-through flow уже реализован для dev. ## Целевая схема @@ -39,7 +39,7 @@ client - Fastify gateway - worker -Gateway уже добавлен как JS/Fastify skeleton. Сейчас `/images/*` возвращает `501`, пока не подключены DB/S3/imgproxy. +Gateway принимает `/images/{assetId}/v{version}/{preset}?w={width}&q={quality}&f=auto`, выбирает фактический формат по `Accept`, вызывает Backend ensure endpoint и кэширует готовые bytes в L1 memory cache. ```bash cp .env.example .env @@ -53,6 +53,32 @@ pnpm gateway:dev pnpm worker:dev ``` +`.env` игнорируется git. Runtime-код не содержит dev credentials fallback: для production нужно передать реальные `DATABASE_URL`, `RABBITMQ_URL`, `S3_*` и `IMGPROXY_UPSTREAM` через окружение. + +Минимальный smoke flow: + +```bash +curl -sS -X POST http://localhost:3001/api/assets \ + -H 'content-type: application/json' \ + -d '{"sourceUrl":"https://storage.yandexcloud.net/shared1318/img/1.jpg","publicId":"asset_demo"}' + +curl -i "http://localhost:8888/images/asset_demo/v1/card?w=640&q=80&f=auto" +curl -i "http://localhost:8888/images/asset_demo/v1/avatar?f=auto" +``` + +Сейчас доступны static presets из `packages/image-config`: `card`, `hero`, `avatar`. Произвольный single-image режим доступен через `/images/{assetId}/v{version}/custom?...`, если включён `IMAGE_ALLOW_CUSTOM_TRANSFORMS=true`. + +Business API без админки: + +```bash +curl -sS http://localhost:3001/api/presets +curl -sS http://localhost:3001/api/assets +curl -sS http://localhost:3001/api/assets/asset_demo/variants +curl -sS -X POST http://localhost:3001/api/assets/asset_demo/variants \ + -H 'content-type: application/json' \ + -d '{"preset":"card","mode":"family"}' +``` + Порты по умолчанию: | Сервис | URL | diff --git a/apps/backend/package.json b/apps/backend/package.json index 697927e..cd531d9 100644 --- a/apps/backend/package.json +++ b/apps/backend/package.json @@ -4,15 +4,21 @@ "private": true, "scripts": { "build": "nest build", - "dev": "nest start --watch", - "start": "node dist/main.js", + "dev": "node --env-file-if-exists=../../.env ./node_modules/@nestjs/cli/bin/nest.js start --watch", + "start": "node --env-file-if-exists=../../.env dist/main.js", "typecheck": "tsc --noEmit -p tsconfig.json" }, "dependencies": { + "@image-platform/database": "workspace:*", + "@image-platform/image-config": "workspace:*", + "@image-platform/queue": "workspace:*", + "@image-platform/storage": "workspace:*", "@nestjs/common": "^11.0.0", "@nestjs/core": "^11.0.0", "@nestjs/platform-express": "^11.0.0", "@nestjs/swagger": "^11.0.0", + "amqplib": "^1.0.4", + "drizzle-orm": "^0.45.2", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.1", "swagger-ui-express": "^5.0.1" @@ -20,6 +26,8 @@ "devDependencies": { "@nestjs/cli": "^11.0.0", "@nestjs/schematics": "^11.0.0", + "@types/amqplib": "^0.10.8", + "@types/express": "^5.0.6", "@types/node": "^24.0.0", "@types/swagger-ui-express": "^4.1.8", "typescript": "^5.9.0" diff --git a/apps/backend/src/app.module.ts b/apps/backend/src/app.module.ts index 87c7eba..3db5ec8 100644 --- a/apps/backend/src/app.module.ts +++ b/apps/backend/src/app.module.ts @@ -1,9 +1,17 @@ import { Module } from "@nestjs/common" +import { AssetsController } from "./assets/assets.controller" +import { AssetsService } from "./assets/assets.service" import { HealthController } from "./health/health.controller" +import { DatabaseService } from "./infra/database.service" +import { QueueService } from "./infra/queue.service" +import { StorageService } from "./infra/storage.service" import { InternalImagesController } from "./internal-images/internal-images.controller" +import { InternalImagesService } from "./internal-images/internal-images.service" +import { PresetsController } from "./presets/presets.controller" @Module({ - controllers: [HealthController, InternalImagesController], + controllers: [HealthController, AssetsController, InternalImagesController, PresetsController], + providers: [AssetsService, DatabaseService, InternalImagesService, QueueService, StorageService], }) export class AppModule {} diff --git a/apps/backend/src/assets/asset-response.dto.ts b/apps/backend/src/assets/asset-response.dto.ts new file mode 100644 index 0000000..3abd20e --- /dev/null +++ b/apps/backend/src/assets/asset-response.dto.ts @@ -0,0 +1,93 @@ +import { ApiProperty, ApiPropertyOptional } from "@nestjs/swagger" + +export class AssetResponseDto { + @ApiProperty({ description: "Внутренний UUID asset.", example: "59fcf4f6-9891-4df4-8bb7-a4dbe570bb66" }) + id!: string + + @ApiProperty({ description: "Публичный идентификатор asset.", example: "asset_demo" }) + publicId!: string + + @ApiProperty({ description: "Текущая версия source image.", example: 1 }) + currentVersion!: number + + @ApiProperty({ description: "Статус asset.", enum: ["active", "disabled", "deleted"], example: "active" }) + status!: string + + @ApiProperty({ description: "Source URL текущей версии.", example: "https://storage.yandexcloud.net/shared1318/img/1.jpg" }) + sourceUrl!: string + + @ApiProperty({ description: "Hostname source URL текущей версии.", example: "storage.yandexcloud.net" }) + sourceHost!: string + + @ApiProperty({ description: "Дата создания asset.", example: "2026-05-05T12:00:00.000Z" }) + createdAt!: string + + @ApiProperty({ description: "Дата обновления asset.", example: "2026-05-05T12:00:00.000Z" }) + updatedAt!: string +} + +export class AssetVariantResponseDto { + @ApiProperty({ description: "Внутренний UUID variant.", example: "7748d24e-5f30-4064-8ee8-4745a4d2aef1" }) + id!: string + + @ApiProperty({ description: "Preset или `custom`.", example: "card" }) + preset!: string + + @ApiProperty({ description: "Версия source image, для которой создан variant.", example: 1 }) + version!: number + + @ApiProperty({ description: "Ширина variant.", example: 640 }) + width!: number + + @ApiProperty({ description: "Высота variant. `0` означает auto height.", example: 0 }) + height!: number + + @ApiProperty({ description: "Режим resize.", enum: ["fit", "fill"], example: "fit" }) + resize!: string + + @ApiProperty({ description: "Качество variant.", example: 80 }) + quality!: number + + @ApiProperty({ description: "Запрошенный формат.", enum: ["auto", "avif", "webp", "jpg", "png"], example: "webp" }) + requestedFormat!: string + + @ApiProperty({ description: "Фактический формат bytes.", enum: ["avif", "webp", "jpg", "png"], example: "webp" }) + format!: string + + @ApiProperty({ description: "Статус генерации.", enum: ["pending", "processing", "ready", "failed"], example: "ready" }) + status!: string + + @ApiProperty({ description: "Публичный Gateway URL для variant.", example: "http://localhost:8888/images/asset_demo/v1/card?w=640&q=80&f=webp" }) + url!: string + + @ApiProperty({ description: "S3 key variant object.", example: "variants/asset_demo/v1/abc.webp" }) + s3Key!: string + + @ApiPropertyOptional({ description: "Content-Type готового object.", example: "image/webp" }) + contentType!: string | null + + @ApiPropertyOptional({ description: "Размер готового object в bytes.", example: 71844 }) + sizeBytes!: number | null + + @ApiPropertyOptional({ description: "Ошибка последней генерации, если status=`failed`." }) + error!: string | null + + @ApiProperty({ description: "Дата создания variant.", example: "2026-05-05T12:00:00.000Z" }) + createdAt!: string + + @ApiProperty({ description: "Дата обновления variant.", example: "2026-05-05T12:00:00.000Z" }) + updatedAt!: string +} + +export class AssetVariantsResponseDto { + @ApiProperty({ description: "Публичный идентификатор asset.", example: "asset_demo" }) + publicId!: string + + @ApiProperty({ description: "Список variants.", type: [AssetVariantResponseDto] }) + variants!: AssetVariantResponseDto[] +} + +export class AssetsListResponseDto { + @ApiProperty({ description: "Список assets.", type: [AssetResponseDto] }) + assets!: AssetResponseDto[] +} diff --git a/apps/backend/src/assets/assets.controller.ts b/apps/backend/src/assets/assets.controller.ts new file mode 100644 index 0000000..a192c4a --- /dev/null +++ b/apps/backend/src/assets/assets.controller.ts @@ -0,0 +1,93 @@ +import { Body, Controller, Get, Param, Post, Query } from "@nestjs/common" +import { + ApiBadRequestResponse, + ApiConflictResponse, + ApiCreatedResponse, + ApiNotFoundResponse, + ApiOkResponse, + ApiOperation, + ApiParam, + ApiQuery, + ApiTags, +} from "@nestjs/swagger" + +import { AssetsService } from "./assets.service" +import { AssetResponseDto, AssetVariantsResponseDto, AssetsListResponseDto } from "./asset-response.dto" +import { CreateAssetVariantsRequestDto, CreateAssetVariantsResponseDto } from "./create-asset-variants.dto" +import { CreateAssetRequestDto, CreateAssetResponseDto } from "./create-asset.dto" + +@ApiTags("assets") +@Controller("assets") +export class AssetsController { + constructor(private readonly assets: AssetsService) {} + + @Get() + @ApiOperation({ + summary: "получить список assets", + description: "Возвращает последние зарегистрированные assets вместе с source URL текущей версии.", + }) + @ApiQuery({ description: "Максимальное количество assets в ответе.", example: 50, name: "limit", required: false }) + @ApiQuery({ description: "Смещение для простого paging.", example: 0, name: "offset", required: false }) + @ApiOkResponse({ description: "Список assets возвращён.", type: AssetsListResponseDto }) + listAssets(@Query("limit") limit?: string, @Query("offset") offset?: string): Promise { + return this.assets.listAssets({ limit, offset }) + } + + @Post() + @ApiOperation({ + summary: "зарегистрировать исходное изображение", + description: + "Создаёт asset и первую версию source image. Source URL сохраняется в PostgreSQL, а публичный image URL строится через Gateway без раскрытия исходной ссылки клиенту.", + }) + @ApiCreatedResponse({ description: "Asset создан, версия source image зарегистрирована.", type: CreateAssetResponseDto }) + @ApiBadRequestResponse({ description: "Некорректный sourceUrl, publicId или source host запрещён настройками." }) + @ApiConflictResponse({ description: "Asset с таким publicId уже существует." }) + createAsset(@Body() request: CreateAssetRequestDto): Promise { + return this.assets.createAsset(request) + } + + @Get(":publicId") + @ApiOperation({ + summary: "получить asset по publicId", + description: "Возвращает metadata asset и source URL текущей версии.", + }) + @ApiParam({ description: "Публичный идентификатор asset.", example: "asset_demo", name: "publicId" }) + @ApiOkResponse({ description: "Asset найден.", type: AssetResponseDto }) + @ApiNotFoundResponse({ description: "Asset не найден." }) + getAsset(@Param("publicId") publicId: string): Promise { + return this.assets.getAsset(publicId) + } + + @Get(":publicId/variants") + @ApiOperation({ + summary: "получить variants asset", + description: "Возвращает variants asset: preset/custom параметры, status, S3 key, public URL и ошибку генерации, если она была.", + }) + @ApiParam({ description: "Публичный идентификатор asset.", example: "asset_demo", name: "publicId" }) + @ApiQuery({ description: "Версия source image. Если не передана, возвращаются variants всех версий.", example: 1, name: "version", required: false }) + @ApiOkResponse({ description: "Variants возвращены.", type: AssetVariantsResponseDto }) + @ApiNotFoundResponse({ description: "Asset не найден." }) + listAssetVariants( + @Param("publicId") publicId: string, + @Query("version") version?: string, + ): Promise { + return this.assets.listAssetVariants(publicId, version) + } + + @Post(":publicId/variants") + @ApiOperation({ + summary: "поставить generation jobs для variants", + description: + "Business endpoint для явной подготовки variants. В режиме `single` создаёт один variant, в режиме `family` создаёт набор variants preset по всем разрешённым widths/formats. Endpoint не ждёт bytes, а возвращает созданные/переиспользованные rows и public URLs.", + }) + @ApiParam({ description: "Публичный идентификатор asset.", example: "asset_demo", name: "publicId" }) + @ApiCreatedResponse({ description: "Variants созданы или переиспользованы, jobs поставлены при необходимости.", type: CreateAssetVariantsResponseDto }) + @ApiBadRequestResponse({ description: "Некорректный preset/custom transform config." }) + @ApiNotFoundResponse({ description: "Asset или version не найдены." }) + createAssetVariants( + @Param("publicId") publicId: string, + @Body() request: CreateAssetVariantsRequestDto, + ): Promise { + return this.assets.createAssetVariants(publicId, request) + } +} diff --git a/apps/backend/src/assets/assets.service.ts b/apps/backend/src/assets/assets.service.ts new file mode 100644 index 0000000..79bc426 --- /dev/null +++ b/apps/backend/src/assets/assets.service.ts @@ -0,0 +1,611 @@ +import { BadRequestException, ConflictException, Injectable, NotFoundException } from "@nestjs/common" +import { imageAssets, imageAssetVersions, imageVariants } from "@image-platform/database" +import { + CUSTOM_PRESET_NAME, + ImageTransformConfigError, + getImagePreset, + isAllowedSourceHost, + loadAllowedSourceHostsFromEnv, + normalizeImageTransform, + parseBooleanFlag, + type ActualImageFormat, + type NormalizedImageTransform, +} from "@image-platform/image-config" +import { buildVariantImageKey } from "@image-platform/storage" +import { and, desc, eq } from "drizzle-orm" +import { createHash, randomUUID } from "node:crypto" + +import { DatabaseService } from "../infra/database.service" +import { QueueService } from "../infra/queue.service" +import type { AssetResponseDto, AssetVariantResponseDto, AssetVariantsResponseDto, AssetsListResponseDto } from "./asset-response.dto" +import type { CreateAssetVariantsRequestDto, CreateAssetVariantsResponseDto } from "./create-asset-variants.dto" +import type { CreateAssetRequestDto, CreateAssetResponseDto } from "./create-asset.dto" +import { normalizeSourceUrl } from "./source-url" + +type AssetVersionRow = { + assetId: string + currentVersion: number + publicId: string + status: "active" | "deleted" | "disabled" + versionId: string + version: number +} + +type VariantRow = typeof imageVariants.$inferSelect + +type VariantTransform = NormalizedImageTransform & { + version: number +} + +@Injectable() +export class AssetsService { + private readonly allowedHosts = loadAllowedSourceHostsFromEnv() + private readonly allowCustomTransforms = parseBooleanFlag(process.env.IMAGE_ALLOW_CUSTOM_TRANSFORMS, false) + private readonly allowUnregisteredHosts = parseBooleanFlag(process.env.SOURCE_HOST_ALLOW_ALL, false) + private readonly publicImageBaseUrl = process.env.PUBLIC_IMAGE_BASE_URL ?? "http://localhost:8888" + + constructor( + private readonly database: DatabaseService, + private readonly queue: QueueService, + ) {} + + async listAssets(input: { limit?: string; offset?: string }): Promise { + const limit = parsePaginationInteger(input.limit, 50, 1, 100) + const offset = parsePaginationInteger(input.offset, 0, 0, 10_000) + + const rows = await this.database.db + .select({ + createdAt: imageAssets.createdAt, + currentVersion: imageAssets.currentVersion, + id: imageAssets.id, + publicId: imageAssets.publicId, + sourceHost: imageAssetVersions.sourceHost, + sourceUrl: imageAssetVersions.sourceUrl, + status: imageAssets.status, + updatedAt: imageAssets.updatedAt, + }) + .from(imageAssets) + .innerJoin( + imageAssetVersions, + and(eq(imageAssetVersions.assetId, imageAssets.id), eq(imageAssetVersions.version, imageAssets.currentVersion)), + ) + .orderBy(desc(imageAssets.createdAt)) + .limit(limit) + .offset(offset) + + return { + assets: rows.map(mapAssetResponse), + } + } + + async createAsset(request: CreateAssetRequestDto): Promise { + const source = normalizeSourceUrl(request.sourceUrl) + await this.assertAllowedHost(source.hostname) + + const publicId = request.publicId ? normalizePublicId(request.publicId) : generatePublicId() + const sourceHash = createHash("sha256").update(source.sourceUrl).digest("hex") + + try { + const result = await this.database.db.transaction(async (tx) => { + const [asset] = await tx + .insert(imageAssets) + .values({ publicId }) + .returning({ id: imageAssets.id, publicId: imageAssets.publicId }) + + if (!asset) { + throw new Error("failed to create image asset") + } + + const [version] = await tx + .insert(imageAssetVersions) + .values({ + assetId: asset.id, + sourceHash, + sourceHost: source.hostname, + sourceUrl: source.sourceUrl, + version: 1, + }) + .returning({ sourceHost: imageAssetVersions.sourceHost, version: imageAssetVersions.version }) + + if (!version) { + throw new Error("failed to create image asset version") + } + + return { asset, version } + }) + + return { + id: result.asset.id, + imageBasePath: `/images/${result.asset.publicId}/v${result.version.version}/card`, + publicId: result.asset.publicId, + sourceHost: result.version.sourceHost, + version: result.version.version, + } + } catch (error) { + if (isUniqueViolation(error)) { + throw new ConflictException("publicId already exists") + } + + throw error + } + } + + async getAsset(publicId: string): Promise { + const asset = await this.loadAsset(publicId) + + return mapAssetResponse(asset) + } + + async listAssetVariants(publicId: string, versionInput?: string): Promise { + const asset = await this.loadAssetVersion(publicId, parseOptionalVersion(versionInput)) + const conditions = [eq(imageVariants.assetId, asset.assetId)] + + if (versionInput !== undefined) { + conditions.push(eq(imageVariants.assetVersion, asset.version)) + } + + const variants = await this.database.db + .select() + .from(imageVariants) + .where(and(...conditions)) + .orderBy(desc(imageVariants.createdAt)) + + return { + publicId: asset.publicId, + variants: variants.map((variant) => this.mapVariantResponse(asset.publicId, variant)), + } + } + + async createAssetVariants( + publicId: string, + request: CreateAssetVariantsRequestDto, + ): Promise { + const asset = await this.loadAssetVersion(publicId, request.version) + const transforms = this.buildVariantTransforms(request, asset.version) + const variants: AssetVariantResponseDto[] = [] + + for (const transform of transforms) { + const variant = await this.findOrCreateVariant(asset, transform) + + if (variant.status === "failed") { + const pending = await this.markVariantPending(variant.id) + this.queue.publishGenerateVariant(pending.id) + variants.push(this.mapVariantResponse(asset.publicId, pending)) + continue + } + + if (variant.status === "pending") { + this.queue.publishGenerateVariant(variant.id) + } + + variants.push(this.mapVariantResponse(asset.publicId, variant)) + } + + return { + publicId: asset.publicId, + variants, + version: asset.version, + } + } + + private async loadAsset(publicId: string) { + const normalizedPublicId = normalizePublicId(publicId) + const [asset] = await this.database.db + .select({ + createdAt: imageAssets.createdAt, + currentVersion: imageAssets.currentVersion, + id: imageAssets.id, + publicId: imageAssets.publicId, + sourceHost: imageAssetVersions.sourceHost, + sourceUrl: imageAssetVersions.sourceUrl, + status: imageAssets.status, + updatedAt: imageAssets.updatedAt, + }) + .from(imageAssets) + .innerJoin( + imageAssetVersions, + and(eq(imageAssetVersions.assetId, imageAssets.id), eq(imageAssetVersions.version, imageAssets.currentVersion)), + ) + .where(eq(imageAssets.publicId, normalizedPublicId)) + .limit(1) + + if (!asset) { + throw new NotFoundException("asset not found") + } + + return asset + } + + private async loadAssetVersion(publicId: string, versionInput: number | null | undefined): Promise { + const normalizedPublicId = normalizePublicId(publicId) + const [asset] = await this.database.db + .select({ + assetId: imageAssets.id, + currentVersion: imageAssets.currentVersion, + publicId: imageAssets.publicId, + status: imageAssets.status, + }) + .from(imageAssets) + .where(eq(imageAssets.publicId, normalizedPublicId)) + .limit(1) + + if (!asset || asset.status !== "active") { + throw new NotFoundException("asset not found") + } + + const version = versionInput ?? asset.currentVersion + const [assetVersion] = await this.database.db + .select({ id: imageAssetVersions.id, version: imageAssetVersions.version }) + .from(imageAssetVersions) + .where(and(eq(imageAssetVersions.assetId, asset.assetId), eq(imageAssetVersions.version, version))) + .limit(1) + + if (!assetVersion) { + throw new NotFoundException("asset version not found") + } + + return { + ...asset, + version: assetVersion.version, + versionId: assetVersion.id, + } + } + + private buildVariantTransforms(request: CreateAssetVariantsRequestDto, version: number): VariantTransform[] { + const mode = request.mode ?? "single" + + if (mode !== "single" && mode !== "family") { + throw new BadRequestException("mode must be single or family") + } + + if (mode === "family") { + return this.buildFamilyTransforms(request, version) + } + + const format = request.format ?? selectDefaultFormat(request.preset, this.allowCustomTransforms) + + return [ + this.normalizeTransform({ + format, + height: request.height, + preset: request.preset, + quality: request.quality, + resize: request.resize, + version, + width: request.width, + }), + ] + } + + private buildFamilyTransforms(request: CreateAssetVariantsRequestDto, version: number): VariantTransform[] { + if (request.preset === CUSTOM_PRESET_NAME) { + throw new BadRequestException("custom transforms do not support family mode") + } + + const preset = getImagePreset(request.preset) + + if (!preset) { + throw new BadRequestException(`unknown image preset: ${request.preset}`) + } + + if (request.width !== undefined || request.height !== undefined || request.resize !== undefined) { + throw new BadRequestException("width, height and resize are not accepted in family mode") + } + + const formats = [...new Set(request.formats?.length ? request.formats : preset.formats)] + + for (const format of formats) { + if (!preset.formats.includes(format)) { + throw new BadRequestException(`format ${format} is not allowed for preset ${request.preset}`) + } + } + + const widths = preset.mode === "fixed" ? [preset.width] : preset.widths + + if (!widths?.length) { + throw new BadRequestException(`preset ${request.preset} has no widths configured`) + } + + return widths.flatMap((width) => + formats.map((format) => + this.normalizeTransform({ + format, + preset: request.preset, + quality: request.quality, + version, + width: preset.mode === "fixed" ? undefined : width, + }), + ), + ) + } + + private normalizeTransform(input: { + format: ActualImageFormat + height?: number + preset: string + quality?: number + resize?: "fill" | "fit" + version: number + width?: number + }): VariantTransform { + try { + const transform = normalizeImageTransform({ + allowCustomTransforms: this.allowCustomTransforms, + format: input.format, + height: input.height, + preset: input.preset, + quality: input.quality, + requestedFormat: input.format, + resize: input.resize, + width: input.width, + }) + + return { + ...transform, + version: input.version, + } + } catch (error) { + if (error instanceof ImageTransformConfigError) { + throw new BadRequestException(error.message) + } + + throw error + } + } + + private async findOrCreateVariant(asset: AssetVersionRow, transform: VariantTransform): Promise { + const existing = await this.findVariant(asset.assetId, transform) + + if (existing) { + return existing + } + + const variantHash = createVariantHash(asset.publicId, transform) + const s3Key = buildVariantImageKey({ + assetId: asset.publicId, + format: transform.format, + variantHash, + version: asset.version, + }) + + const [created] = await this.database.db + .insert(imageVariants) + .values({ + assetId: asset.assetId, + assetVersion: asset.version, + assetVersionId: asset.versionId, + format: transform.format, + height: transform.height, + preset: transform.preset, + quality: transform.quality, + requestedFormat: transform.requestedFormat, + resizeMode: transform.resize, + s3Key, + status: "pending", + variantHash, + width: transform.width, + }) + .onConflictDoNothing({ + target: [ + imageVariants.assetId, + imageVariants.assetVersion, + imageVariants.preset, + imageVariants.width, + imageVariants.height, + imageVariants.resizeMode, + imageVariants.quality, + imageVariants.format, + ], + }) + .returning() + + if (created) { + return created + } + + const raced = await this.findVariant(asset.assetId, transform) + + if (!raced) { + throw new Error("failed to create image variant") + } + + return raced + } + + private async findVariant(assetId: string, transform: VariantTransform): Promise { + const [variant] = await this.database.db + .select() + .from(imageVariants) + .where( + and( + eq(imageVariants.assetId, assetId), + eq(imageVariants.assetVersion, transform.version), + eq(imageVariants.preset, transform.preset), + eq(imageVariants.width, transform.width), + eq(imageVariants.height, transform.height), + eq(imageVariants.resizeMode, transform.resize), + eq(imageVariants.quality, transform.quality), + eq(imageVariants.format, transform.format), + ), + ) + .limit(1) + + return variant ?? null + } + + private async markVariantPending(variantId: string): Promise { + const [variant] = await this.database.db + .update(imageVariants) + .set({ error: null, status: "pending", updatedAt: new Date() }) + .where(eq(imageVariants.id, variantId)) + .returning() + + if (!variant) { + throw new NotFoundException("variant not found") + } + + return variant + } + + private mapVariantResponse(publicId: string, variant: VariantRow): AssetVariantResponseDto { + return { + contentType: variant.contentType, + createdAt: variant.createdAt.toISOString(), + error: variant.error, + format: variant.format, + height: variant.height ?? 0, + id: variant.id, + preset: variant.preset, + quality: variant.quality, + requestedFormat: variant.requestedFormat, + resize: variant.resizeMode, + s3Key: variant.s3Key, + sizeBytes: variant.sizeBytes, + status: variant.status, + updatedAt: variant.updatedAt.toISOString(), + url: buildPublicImageUrl(this.publicImageBaseUrl, publicId, variant), + version: variant.assetVersion, + width: variant.width, + } + } + + private async assertAllowedHost(hostname: string) { + if (this.allowUnregisteredHosts) { + return + } + + if (!isAllowedSourceHost(hostname, this.allowedHosts)) { + throw new BadRequestException("sourceUrl host is not allowed") + } + } +} + +function mapAssetResponse(row: { + createdAt: Date + currentVersion: number + id: string + publicId: string + sourceHost: string + sourceUrl: string + status: "active" | "deleted" | "disabled" + updatedAt: Date +}): AssetResponseDto { + return { + createdAt: row.createdAt.toISOString(), + currentVersion: row.currentVersion, + id: row.id, + publicId: row.publicId, + sourceHost: row.sourceHost, + sourceUrl: row.sourceUrl, + status: row.status, + updatedAt: row.updatedAt.toISOString(), + } +} + +function generatePublicId() { + return `asset_${randomUUID().replaceAll("-", "").slice(0, 16)}` +} + +function normalizePublicId(publicId: string) { + const normalized = publicId.trim() + + if (!/^[a-zA-Z0-9_-]{3,128}$/.test(normalized)) { + throw new BadRequestException("publicId must be 3-128 chars and contain only letters, digits, _ or -") + } + + return normalized +} + +function isUniqueViolation(error: unknown) { + return typeof error === "object" && error !== null && "code" in error && error.code === "23505" +} + +function parsePaginationInteger(value: string | undefined, fallback: number, min: number, max: number) { + if (value === undefined) { + return fallback + } + + if (!/^\d+$/.test(value)) { + throw new BadRequestException("pagination params must be integers") + } + + const parsed = Number.parseInt(value, 10) + + if (!Number.isSafeInteger(parsed) || parsed < min || parsed > max) { + throw new BadRequestException(`pagination param must be between ${min} and ${max}`) + } + + return parsed +} + +function parseOptionalVersion(value: string | undefined) { + if (value === undefined) { + return undefined + } + + if (!/^\d+$/.test(value)) { + throw new BadRequestException("version must be a positive integer") + } + + const parsed = Number.parseInt(value, 10) + + if (!Number.isSafeInteger(parsed) || parsed < 1) { + throw new BadRequestException("version must be a positive integer") + } + + return parsed +} + +function selectDefaultFormat(preset: string, allowCustomTransforms: boolean): ActualImageFormat { + const config = getImagePreset(preset) + + if (config) { + return config.formats.includes("webp") ? "webp" : config.formats[0] + } + + if (preset === CUSTOM_PRESET_NAME && allowCustomTransforms) { + return "webp" + } + + throw new BadRequestException(`unknown image preset: ${preset}`) +} + +function createVariantHash(publicId: string, transform: VariantTransform) { + return createHash("sha256") + .update( + [ + publicId, + transform.version, + transform.preset, + transform.width, + transform.height, + transform.resize, + transform.quality, + transform.format, + ].join(":"), + ) + .digest("hex") + .slice(0, 32) +} + +function buildPublicImageUrl(baseUrl: string, publicId: string, variant: VariantRow) { + const url = new URL(`/images/${publicId}/v${variant.assetVersion}/${variant.preset}`, baseUrl) + const isFixedPresetUrl = variant.preset !== CUSTOM_PRESET_NAME && variant.height && variant.height > 0 + + if (!isFixedPresetUrl || variant.preset === CUSTOM_PRESET_NAME) { + url.searchParams.set("w", variant.width.toString()) + } + + if (variant.preset === CUSTOM_PRESET_NAME && variant.height) { + url.searchParams.set("h", variant.height.toString()) + url.searchParams.set("fit", variant.resizeMode) + } + + if (!isFixedPresetUrl || variant.quality !== 80) { + url.searchParams.set("q", variant.quality.toString()) + } + + url.searchParams.set("f", variant.format) + + return url.toString() +} diff --git a/apps/backend/src/assets/create-asset-variants.dto.ts b/apps/backend/src/assets/create-asset-variants.dto.ts new file mode 100644 index 0000000..1539026 --- /dev/null +++ b/apps/backend/src/assets/create-asset-variants.dto.ts @@ -0,0 +1,43 @@ +import { ApiProperty, ApiPropertyOptional } from "@nestjs/swagger" + +import { AssetVariantResponseDto } from "./asset-response.dto" + +export class CreateAssetVariantsRequestDto { + @ApiProperty({ description: "Preset для генерации или `custom`.", example: "card" }) + preset!: string + + @ApiPropertyOptional({ description: "Режим генерации: один variant или вся family preset.", enum: ["single", "family"], example: "single" }) + mode?: "family" | "single" + + @ApiPropertyOptional({ description: "Версия source image. Если не передана, используется currentVersion asset.", example: 1 }) + version?: number + + @ApiPropertyOptional({ description: "Ширина variant. Обязательна для responsive preset в mode=`single` и custom.", example: 640 }) + width?: number + + @ApiPropertyOptional({ description: "Высота variant для custom. `0` или отсутствие означает auto height.", example: 333 }) + height?: number + + @ApiPropertyOptional({ description: "Качество. Если не передано, берётся из preset/custom config.", example: 80 }) + quality?: number + + @ApiPropertyOptional({ description: "Фактический формат для single generation.", enum: ["avif", "webp", "jpg", "png"], example: "webp" }) + format?: "avif" | "jpg" | "png" | "webp" + + @ApiPropertyOptional({ description: "Форматы для family generation. Если не переданы, используются все форматы preset.", enum: ["avif", "webp", "jpg", "png"], isArray: true }) + formats?: Array<"avif" | "jpg" | "png" | "webp"> + + @ApiPropertyOptional({ description: "Resize mode для custom transforms.", enum: ["fit", "fill"], example: "fill" }) + resize?: "fill" | "fit" +} + +export class CreateAssetVariantsResponseDto { + @ApiProperty({ description: "Публичный идентификатор asset.", example: "asset_demo" }) + publicId!: string + + @ApiProperty({ description: "Версия source image, для которой поставлены jobs.", example: 1 }) + version!: number + + @ApiProperty({ description: "Созданные или переиспользованные variants.", type: [AssetVariantResponseDto] }) + variants!: AssetVariantResponseDto[] +} diff --git a/apps/backend/src/assets/create-asset.dto.ts b/apps/backend/src/assets/create-asset.dto.ts new file mode 100644 index 0000000..8b68371 --- /dev/null +++ b/apps/backend/src/assets/create-asset.dto.ts @@ -0,0 +1,36 @@ +import { ApiProperty, ApiPropertyOptional } from "@nestjs/swagger" + +export class CreateAssetRequestDto { + @ApiPropertyOptional({ + description: + "Публичный стабильный идентификатор asset. Если не передан, Backend сгенерирует идентификатор автоматически.", + example: "asset_123", + }) + publicId?: string + + @ApiProperty({ + description: "Постоянная ссылка на исходное изображение. Сейчас поддерживаются только публичные http/https URL.", + example: "https://storage.yandexcloud.net/shared1318/img/1.jpg", + }) + sourceUrl!: string +} + +export class CreateAssetResponseDto { + @ApiProperty({ description: "Внутренний UUID asset в PostgreSQL.", example: "59fcf4f6-9891-4df4-8bb7-a4dbe570bb66" }) + id!: string + + @ApiProperty({ description: "Публичный идентификатор asset для Gateway URL.", example: "asset_123" }) + publicId!: string + + @ApiProperty({ description: "Номер версии source image. Используется в URL как `v{version}`.", example: 1 }) + version!: number + + @ApiProperty({ description: "Нормализованный hostname исходного изображения.", example: "storage.yandexcloud.net" }) + sourceHost!: string + + @ApiProperty({ + description: "Базовый путь Gateway для запроса variant. Width, quality и format передаются query params.", + example: "/images/asset_123/v1/card", + }) + imageBasePath!: string +} diff --git a/apps/backend/src/assets/source-url.ts b/apps/backend/src/assets/source-url.ts new file mode 100644 index 0000000..39f7274 --- /dev/null +++ b/apps/backend/src/assets/source-url.ts @@ -0,0 +1,65 @@ +import { BadRequestException } from "@nestjs/common" +import { isIP } from "node:net" + +const LOCAL_HOSTNAMES = new Set(["localhost", "localhost.localdomain"]) + +export type NormalizedSourceUrl = { + hostname: string + sourceUrl: string +} + +export function normalizeSourceUrl(input: string): NormalizedSourceUrl { + let url: URL + + try { + url = new URL(input) + } catch { + throw new BadRequestException("sourceUrl must be a valid URL") + } + + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new BadRequestException("sourceUrl must use http or https") + } + + const hostname = url.hostname.toLowerCase().replace(/\.$/, "") + + if (isPrivateOrLocalHostname(hostname)) { + throw new BadRequestException("sourceUrl host must be public") + } + + url.hostname = hostname + url.hash = "" + + return { + hostname, + sourceUrl: url.toString(), + } +} + +function isPrivateOrLocalHostname(hostname: string) { + if (LOCAL_HOSTNAMES.has(hostname) || hostname.endsWith(".localhost")) { + return true + } + + const ipVersion = isIP(hostname) + + if (ipVersion === 0) { + return false + } + + if (ipVersion === 6) { + return true + } + + const [a = 0, b = 0] = hostname.split(".").map((part) => Number.parseInt(part, 10)) + + return ( + a === 0 || + a === 10 || + a === 127 || + (a === 100 && b >= 64 && b <= 127) || + (a === 169 && b === 254) || + (a === 172 && b >= 16 && b <= 31) || + (a === 192 && b === 168) + ) +} diff --git a/apps/backend/src/health/health-response.dto.ts b/apps/backend/src/health/health-response.dto.ts index 2f08316..38f07e1 100644 --- a/apps/backend/src/health/health-response.dto.ts +++ b/apps/backend/src/health/health-response.dto.ts @@ -1,9 +1,9 @@ import { ApiProperty } from "@nestjs/swagger" export class HealthResponseDto { - @ApiProperty({ example: "image-platform-api" }) + @ApiProperty({ description: "Имя сервиса, который вернул health-check ответ.", example: "image-platform-api" }) service!: string - @ApiProperty({ example: "ok" }) + @ApiProperty({ description: "Текущее состояние сервиса.", example: "ok" }) status!: string } diff --git a/apps/backend/src/health/health.controller.ts b/apps/backend/src/health/health.controller.ts index c2d742f..45701f5 100644 --- a/apps/backend/src/health/health.controller.ts +++ b/apps/backend/src/health/health.controller.ts @@ -7,8 +7,11 @@ import { HealthResponseDto } from "./health-response.dto" @Controller("health") export class HealthController { @Get() - @ApiOperation({ summary: "Проверить состояние API" }) - @ApiOkResponse({ type: HealthResponseDto }) + @ApiOperation({ + summary: "проверить состояние Backend API", + description: "Возвращает простой health-check ответ. Используется для локальной проверки, мониторинга и smoke tests.", + }) + @ApiOkResponse({ description: "Backend API доступен и отвечает.", type: HealthResponseDto }) getHealth(): HealthResponseDto { return { service: "image-platform-api", diff --git a/apps/backend/src/infra/database.service.ts b/apps/backend/src/infra/database.service.ts new file mode 100644 index 0000000..398c69a --- /dev/null +++ b/apps/backend/src/infra/database.service.ts @@ -0,0 +1,14 @@ +import { Injectable, OnModuleDestroy } from "@nestjs/common" +import { createDatabase, createDatabasePool } from "@image-platform/database" +import type { Database } from "@image-platform/database" + +@Injectable() +export class DatabaseService implements OnModuleDestroy { + private readonly pool = createDatabasePool() + + readonly db: Database = createDatabase(this.pool) + + async onModuleDestroy() { + await this.pool.end() + } +} diff --git a/apps/backend/src/infra/queue.service.ts b/apps/backend/src/infra/queue.service.ts new file mode 100644 index 0000000..d3b20b8 --- /dev/null +++ b/apps/backend/src/infra/queue.service.ts @@ -0,0 +1,52 @@ +import { Injectable, OnModuleDestroy, OnModuleInit, ServiceUnavailableException } from "@nestjs/common" +import amqp, { type Channel, type ChannelModel } from "amqplib" +import { assertQueueTopology, loadQueueTopologyFromEnv, publishGenerateVariantJob } from "@image-platform/queue" +import { randomUUID } from "node:crypto" + +@Injectable() +export class QueueService implements OnModuleDestroy, OnModuleInit { + private readonly rabbitmqUrl = getRequiredEnv("RABBITMQ_URL") + private readonly topology = loadQueueTopologyFromEnv() + private channel: Channel | null = null + private connection: ChannelModel | null = null + + async onModuleInit() { + const connection = await amqp.connect(this.rabbitmqUrl) + const channel = await connection.createChannel() + + await assertQueueTopology(channel, this.topology) + + this.connection = connection + this.channel = channel + } + + async onModuleDestroy() { + await this.channel?.close().catch((error: unknown) => console.error("failed to close RabbitMQ channel", error)) + await this.connection?.close().catch((error: unknown) => console.error("failed to close RabbitMQ connection", error)) + } + + publishGenerateVariant(variantId: string) { + if (!this.channel) { + throw new ServiceUnavailableException("RabbitMQ channel is not ready") + } + + const published = publishGenerateVariantJob(this.channel, this.topology, { + jobId: randomUUID(), + variantId, + }) + + if (!published) { + throw new ServiceUnavailableException("RabbitMQ publish buffer is full") + } + } +} + +function getRequiredEnv(name: string) { + const value = process.env[name] + + if (!value) { + throw new Error(`${name} is required`) + } + + return value +} diff --git a/apps/backend/src/infra/storage.service.ts b/apps/backend/src/infra/storage.service.ts new file mode 100644 index 0000000..5d21e9e --- /dev/null +++ b/apps/backend/src/infra/storage.service.ts @@ -0,0 +1,14 @@ +import { Injectable } from "@nestjs/common" +import { createS3Client, getObjectBuffer, loadStorageConfigFromEnv, type StoredObject } from "@image-platform/storage" + +@Injectable() +export class StorageService { + private readonly config = loadStorageConfigFromEnv() + private readonly client = createS3Client(this.config) + + readonly bucket = this.config.bucket + + async getObject(key: string): Promise { + return getObjectBuffer(this.client, this.bucket, key) + } +} diff --git a/apps/backend/src/internal-images/ensure-image-variant.dto.ts b/apps/backend/src/internal-images/ensure-image-variant.dto.ts index 3129fb4..497d9b4 100644 --- a/apps/backend/src/internal-images/ensure-image-variant.dto.ts +++ b/apps/backend/src/internal-images/ensure-image-variant.dto.ts @@ -1,21 +1,43 @@ -import { ApiProperty } from "@nestjs/swagger" +import { ApiProperty, ApiPropertyOptional } from "@nestjs/swagger" +import type { ActualImageFormat, RequestedImageFormat, ResizeMode } from "@image-platform/image-config" export class EnsureImageVariantRequestDto { - @ApiProperty({ example: "asset_123" }) + @ApiProperty({ description: "Публичный идентификатор asset из Gateway URL.", example: "asset_123" }) assetId!: string - @ApiProperty({ example: 4, minimum: 1 }) + @ApiProperty({ description: "Версия source image из Gateway URL `v{version}`.", example: 4, minimum: 1 }) version!: number - @ApiProperty({ example: "card" }) + @ApiProperty({ description: "Имя preset трансформации. Сейчас используется как часть variant key.", example: "card" }) preset!: string - @ApiProperty({ example: 640, minimum: 1 }) - width!: number + @ApiPropertyOptional({ description: "Целевая ширина variant в пикселях. Обязательна для responsive presets и custom.", example: 640, minimum: 1 }) + width?: number - @ApiProperty({ example: 80, minimum: 1 }) - quality!: number + @ApiPropertyOptional({ description: "Целевая высота variant в пикселях. `0` или отсутствие означает auto height.", example: 420, minimum: 0 }) + height?: number - @ApiProperty({ enum: ["auto", "avif", "webp", "jpg", "png"], example: "auto" }) - format!: "auto" | "avif" | "jpg" | "png" | "webp" + @ApiPropertyOptional({ description: "Качество сжатия для imgproxy. Если не передано, берётся из preset.", example: 80, minimum: 1 }) + quality?: number + + @ApiPropertyOptional({ + description: "Формат, который запросил клиент. Для `auto` Gateway выбирает фактический формат по `Accept` header.", + enum: ["auto", "avif", "webp", "jpg", "png"], + example: "auto", + }) + requestedFormat?: RequestedImageFormat + + @ApiPropertyOptional({ + description: "Режим resize для custom transforms. Для обычных presets берётся из preset config.", + enum: ["fit", "fill"], + example: "fit", + }) + resize?: ResizeMode + + @ApiProperty({ + description: "Фактический output format после negotiation. Именно этот формат попадает в S3 key и L1 cache key.", + enum: ["avif", "webp", "jpg", "png"], + example: "webp", + }) + format!: ActualImageFormat } diff --git a/apps/backend/src/internal-images/internal-images.controller.ts b/apps/backend/src/internal-images/internal-images.controller.ts index 9216d7a..2c52ec7 100644 --- a/apps/backend/src/internal-images/internal-images.controller.ts +++ b/apps/backend/src/internal-images/internal-images.controller.ts @@ -1,19 +1,63 @@ -import { Body, Controller, NotImplementedException, Post } from "@nestjs/common" -import { ApiOperation, ApiResponse, ApiTags } from "@nestjs/swagger" +import { Body, Controller, Header, Post, Res, StreamableFile } from "@nestjs/common" +import { + ApiBadGatewayResponse, + ApiBadRequestResponse, + ApiNotFoundResponse, + ApiOkResponse, + ApiOperation, + ApiResponse, + ApiTags, +} from "@nestjs/swagger" +import type { Response } from "express" import { EnsureImageVariantRequestDto } from "./ensure-image-variant.dto" +import { InternalImagesService } from "./internal-images.service" @ApiTags("internal-images") @Controller("internal/images") export class InternalImagesController { + constructor(private readonly internalImages: InternalImagesService) {} + @Post("ensure") - @ApiOperation({ summary: "Ensure image variant for Gateway L1 miss" }) - @ApiResponse({ status: 501, description: "Read-through image pipeline is not implemented yet" }) - ensureImageVariant(@Body() request: EnsureImageVariantRequestDto): never { - throw new NotImplementedException({ - message: "image read-through pipeline is not implemented yet", - request, - status: "not_implemented", - }) + @ApiOperation({ + summary: "подготовить variant изображения для Gateway", + description: + "Внутренний endpoint для Gateway. На L1 cache miss Backend проверяет PostgreSQL и S3, создаёт variant при необходимости, публикует RabbitMQ job, ждёт генерацию worker и возвращает готовые image bytes.", + }) + @ApiOkResponse({ + description: "Variant уже был готов в S3 или был успешно сгенерирован worker.", + content: { + "image/*": { + schema: { + format: "binary", + type: "string", + }, + }, + }, + }) + @ApiBadRequestResponse({ description: "Некорректный assetId, version, preset, width, quality или format." }) + @ApiNotFoundResponse({ description: "Asset или указанная версия source image не найдены." }) + @ApiBadGatewayResponse({ description: "Worker/imgproxy/S3 не смогли подготовить или вернуть variant." }) + @ApiResponse({ status: 504, description: "Variant не успел сгенерироваться до истечения IMAGE_ENSURE_WAIT_MS." }) + @Header("Cache-Control", "public, max-age=31536000, immutable") + async ensureImageVariant( + @Body() request: EnsureImageVariantRequestDto, + @Res({ passthrough: true }) response: Response, + ): Promise { + const result = await this.internalImages.ensureImageVariant(request) + + response.setHeader("Cache-Control", result.cacheControl) + response.setHeader("Content-Length", result.contentLength.toString()) + response.setHeader("Content-Type", result.contentType) + + if (result.etag) { + response.setHeader("ETag", result.etag) + } + + if (result.vary) { + response.setHeader("Vary", result.vary) + } + + return new StreamableFile(result.body) } } diff --git a/apps/backend/src/internal-images/internal-images.service.ts b/apps/backend/src/internal-images/internal-images.service.ts new file mode 100644 index 0000000..f862307 --- /dev/null +++ b/apps/backend/src/internal-images/internal-images.service.ts @@ -0,0 +1,326 @@ +import { + BadGatewayException, + BadRequestException, + GatewayTimeoutException, + Injectable, + NotFoundException, +} from "@nestjs/common" +import { imageAssetVersions, imageAssets, imageVariants } from "@image-platform/database" +import { + ImageTransformConfigError, + normalizeImageTransform, + parseBooleanFlag, + type ActualImageFormat, + type NormalizedImageTransform, + type RequestedImageFormat, +} from "@image-platform/image-config" +import { buildVariantImageKey } from "@image-platform/storage" +import { and, eq } from "drizzle-orm" +import { createHash } from "node:crypto" + +import { DatabaseService } from "../infra/database.service" +import { QueueService } from "../infra/queue.service" +import { StorageService } from "../infra/storage.service" +import type { EnsureImageVariantRequestDto } from "./ensure-image-variant.dto" + +type NormalizedEnsureRequest = NormalizedImageTransform & { + assetId: string + version: number +} + +type VariantRow = typeof imageVariants.$inferSelect + +export type EnsuredImageVariant = { + body: Buffer + cacheControl: string + contentLength: number + contentType: string + etag: string | null + vary: string | null +} + +@Injectable() +export class InternalImagesService { + private readonly ensureWaitMs = parsePositiveInteger(process.env.IMAGE_ENSURE_WAIT_MS, 15_000) + + constructor( + private readonly database: DatabaseService, + private readonly queue: QueueService, + private readonly storage: StorageService, + ) {} + + async ensureImageVariant(request: EnsureImageVariantRequestDto): Promise { + const normalized = normalizeRequest(request) + const assetVersion = await this.loadAssetVersion(normalized) + let variant = await this.findOrCreateVariant(normalized, assetVersion) + + if (variant.status === "ready") { + const ready = await this.loadReadyObject(variant, normalized.requestedFormat) + + if (ready) { + return ready + } + + variant = await this.markVariantPending(variant.id) + } + + if (variant.status === "failed") { + variant = await this.markVariantPending(variant.id) + } + + this.queue.publishGenerateVariant(variant.id) + + const readyVariant = await this.waitForReadyVariant(variant.id) + const ready = await this.loadReadyObject(readyVariant, normalized.requestedFormat) + + if (!ready) { + throw new BadGatewayException("variant was marked ready but S3 object is missing") + } + + return ready + } + + private async loadAssetVersion(request: NormalizedEnsureRequest) { + const [row] = await this.database.db + .select({ + assetId: imageAssets.id, + assetStatus: imageAssets.status, + assetVersionId: imageAssetVersions.id, + }) + .from(imageAssets) + .innerJoin(imageAssetVersions, eq(imageAssetVersions.assetId, imageAssets.id)) + .where(and(eq(imageAssets.publicId, request.assetId), eq(imageAssetVersions.version, request.version))) + .limit(1) + + if (!row || row.assetStatus !== "active") { + throw new NotFoundException("image asset version not found") + } + + return row + } + + private async findOrCreateVariant( + request: NormalizedEnsureRequest, + assetVersion: { assetId: string; assetVersionId: string }, + ): Promise { + const existing = await this.findVariant(request, assetVersion.assetId) + + if (existing) { + return existing + } + + const variantHash = createVariantHash(request) + const s3Key = buildVariantImageKey({ + assetId: request.assetId, + format: request.format, + variantHash, + version: request.version, + }) + + const [created] = await this.database.db + .insert(imageVariants) + .values({ + assetId: assetVersion.assetId, + assetVersion: request.version, + assetVersionId: assetVersion.assetVersionId, + format: request.format, + height: request.height, + preset: request.preset, + quality: request.quality, + requestedFormat: request.requestedFormat, + resizeMode: request.resize, + s3Key, + status: "pending", + variantHash, + width: request.width, + }) + .onConflictDoNothing({ + target: [ + imageVariants.assetId, + imageVariants.assetVersion, + imageVariants.preset, + imageVariants.width, + imageVariants.height, + imageVariants.resizeMode, + imageVariants.quality, + imageVariants.format, + ], + }) + .returning() + + if (created) { + return created + } + + const raced = await this.findVariant(request, assetVersion.assetId) + + if (!raced) { + throw new Error("failed to create image variant") + } + + return raced + } + + private async findVariant(request: NormalizedEnsureRequest, assetId: string) { + const [variant] = await this.database.db + .select() + .from(imageVariants) + .where( + and( + eq(imageVariants.assetId, assetId), + eq(imageVariants.assetVersion, request.version), + eq(imageVariants.preset, request.preset), + eq(imageVariants.width, request.width), + eq(imageVariants.height, request.height), + eq(imageVariants.resizeMode, request.resize), + eq(imageVariants.quality, request.quality), + eq(imageVariants.format, request.format), + ), + ) + .limit(1) + + return variant ?? null + } + + private async markVariantPending(variantId: string) { + const [variant] = await this.database.db + .update(imageVariants) + .set({ error: null, status: "pending", updatedAt: new Date() }) + .where(eq(imageVariants.id, variantId)) + .returning() + + if (!variant) { + throw new NotFoundException("image variant not found") + } + + return variant + } + + private async loadReadyObject( + variant: VariantRow, + requestedFormat: RequestedImageFormat, + ): Promise { + const object = await this.storage.getObject(variant.s3Key) + + if (!object) { + return null + } + + await this.database.db + .update(imageVariants) + .set({ lastAccessedAt: new Date(), updatedAt: new Date() }) + .where(eq(imageVariants.id, variant.id)) + + return { + body: object.body, + cacheControl: "public, max-age=31536000, immutable", + contentLength: object.contentLength ?? object.body.length, + contentType: object.contentType ?? contentTypeForFormat(variant.format), + etag: object.etag, + vary: requestedFormat === "auto" ? "Accept" : null, + } + } + + private async waitForReadyVariant(variantId: string) { + const deadline = Date.now() + this.ensureWaitMs + + while (Date.now() <= deadline) { + const [variant] = await this.database.db.select().from(imageVariants).where(eq(imageVariants.id, variantId)).limit(1) + + if (!variant) { + throw new NotFoundException("image variant not found") + } + + if (variant.status === "ready") { + return variant + } + + if (variant.status === "failed") { + throw new BadGatewayException(variant.error ?? "image variant generation failed") + } + + await sleep(200) + } + + throw new GatewayTimeoutException("image variant generation timed out") + } +} + +function normalizeRequest(request: EnsureImageVariantRequestDto): NormalizedEnsureRequest { + if (!/^[a-zA-Z0-9_-]{3,128}$/.test(request.assetId)) { + throw new BadRequestException("assetId is invalid") + } + + if (!isPositiveInteger(request.version)) { + throw new BadRequestException("version must be a positive integer") + } + + try { + const transform = normalizeImageTransform({ + allowCustomTransforms: parseBooleanFlag(process.env.IMAGE_ALLOW_CUSTOM_TRANSFORMS, false), + format: request.format, + height: request.height, + preset: request.preset, + quality: request.quality, + requestedFormat: request.requestedFormat, + resize: request.resize, + width: request.width, + }) + + return { + ...transform, + assetId: request.assetId, + version: request.version, + } + } catch (error) { + if (error instanceof ImageTransformConfigError) { + throw new BadRequestException(error.message) + } + + throw error + } +} + +function createVariantHash(request: NormalizedEnsureRequest) { + return createHash("sha256") + .update( + [ + request.assetId, + request.version, + request.preset, + request.width, + request.height, + request.resize, + request.quality, + request.format, + ].join(":"), + ) + .digest("hex") + .slice(0, 32) +} + +function contentTypeForFormat(format: ActualImageFormat) { + if (format === "jpg") { + return "image/jpeg" + } + + return `image/${format}` +} + +function isPositiveInteger(value: number) { + return Number.isSafeInteger(value) && value > 0 +} + +function parsePositiveInteger(value: string | undefined, fallback: number) { + if (!value) { + return fallback + } + + const parsed = Number.parseInt(value, 10) + + return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback +} + +function sleep(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)) +} diff --git a/apps/backend/src/main.ts b/apps/backend/src/main.ts index bbc4dc2..8878bcb 100644 --- a/apps/backend/src/main.ts +++ b/apps/backend/src/main.ts @@ -11,13 +11,16 @@ async function bootstrap() { const openApiConfig = new DocumentBuilder() .setTitle("Image Platform API") - .setDescription("Control plane for image assets, variants, S3 storage and external imgproxy.") + .setDescription( + "Backend API для управления image assets, metadata в PostgreSQL, S3 variants, RabbitMQ jobs и генерацией через imgproxy.", + ) .setVersion("0.1.0") - .addTag("system") - .addTag("assets") - .addTag("variants") - .addTag("allowed-hosts") - .addTag("internal-images") + .addTag("system", "Системные endpoints для проверки состояния сервиса.") + .addTag("assets", "Регистрация и управление исходными изображениями.") + .addTag("variants", "Будущие endpoints для управления производными версиями изображений.") + .addTag("allowed-hosts", "Будущие endpoints для управления разрешёнными source hosts.") + .addTag("internal-images", "Внутренние endpoints, которые вызывает Gateway на cache miss.") + .addTag("presets", "Статические presets, custom limits и mock allowlist source hosts.") .build() const openApiDocument = SwaggerModule.createDocument(app, openApiConfig) @@ -29,7 +32,7 @@ async function bootstrap() { }, }) - const port = Number.parseInt(process.env.API_PORT ?? "3001", 10) + const port = Number.parseInt(process.env.BACKEND_PORT ?? process.env.API_PORT ?? "3001", 10) await app.listen(port) } diff --git a/apps/backend/src/presets/presets.controller.ts b/apps/backend/src/presets/presets.controller.ts new file mode 100644 index 0000000..58fa01a --- /dev/null +++ b/apps/backend/src/presets/presets.controller.ts @@ -0,0 +1,49 @@ +import { Controller, Get } from "@nestjs/common" +import { ApiOkResponse, ApiOperation, ApiTags } from "@nestjs/swagger" +import { + CUSTOM_TRANSFORM_CONFIG, + IMAGE_PRESETS, + loadAllowedSourceHostsFromEnv, + parseBooleanFlag, + type ImagePreset, +} from "@image-platform/image-config" + +import { PresetsResponseDto } from "./presets.dto" + +@ApiTags("presets") +@Controller("presets") +export class PresetsController { + @Get() + @ApiOperation({ + summary: "получить доступные presets и custom config", + description: "Возвращает статический config presets, custom transform limits и mock allowlist source hosts.", + }) + @ApiOkResponse({ description: "Конфигурация presets возвращена.", type: PresetsResponseDto }) + getPresets(): PresetsResponseDto { + return { + allowedSourceHosts: [...loadAllowedSourceHostsFromEnv()], + custom: { + enabled: parseBooleanFlag(process.env.IMAGE_ALLOW_CUSTOM_TRANSFORMS, false), + formats: CUSTOM_TRANSFORM_CONFIG.formats, + maxHeight: CUSTOM_TRANSFORM_CONFIG.maxHeight, + maxWidth: CUSTOM_TRANSFORM_CONFIG.maxWidth, + quality: CUSTOM_TRANSFORM_CONFIG.quality, + }, + presets: Object.entries(IMAGE_PRESETS).map(([name, preset]) => { + const config: ImagePreset = preset + + return { + formats: config.formats, + height: config.height, + mode: config.mode, + name, + qualities: config.qualities, + quality: config.quality, + resize: config.resize, + width: config.width, + widths: config.widths, + } + }), + } + } +} diff --git a/apps/backend/src/presets/presets.dto.ts b/apps/backend/src/presets/presets.dto.ts new file mode 100644 index 0000000..d8f6e05 --- /dev/null +++ b/apps/backend/src/presets/presets.dto.ts @@ -0,0 +1,58 @@ +import { ApiProperty } from "@nestjs/swagger" + +export class PresetResponseDto { + @ApiProperty({ description: "Имя preset.", example: "card" }) + name!: string + + @ApiProperty({ description: "Режим preset.", enum: ["fixed", "responsive"], example: "responsive" }) + mode!: string + + @ApiProperty({ description: "Разрешённые форматы.", example: ["avif", "webp", "jpg"] }) + formats!: readonly string[] + + @ApiProperty({ description: "Разрешённые значения quality.", example: [75, 80] }) + qualities!: readonly number[] + + @ApiProperty({ description: "Quality по умолчанию.", example: 80 }) + quality!: number + + @ApiProperty({ description: "Resize mode preset.", enum: ["fit", "fill"], example: "fit" }) + resize!: string + + @ApiProperty({ description: "Фиксированная ширина для fixed preset.", example: 256, required: false }) + width?: number + + @ApiProperty({ description: "Фиксированная высота для fixed preset.", example: 256, required: false }) + height?: number + + @ApiProperty({ description: "Разрешённые ширины для responsive preset.", example: [320, 640, 960], required: false }) + widths?: readonly number[] +} + +export class CustomTransformConfigResponseDto { + @ApiProperty({ description: "Включены ли custom transforms.", example: true }) + enabled!: boolean + + @ApiProperty({ description: "Разрешённые форматы custom transforms.", example: ["avif", "webp", "jpg", "png"] }) + formats!: readonly string[] + + @ApiProperty({ description: "Максимальная ширина custom transform.", example: 4096 }) + maxWidth!: number + + @ApiProperty({ description: "Максимальная высота custom transform.", example: 4096 }) + maxHeight!: number + + @ApiProperty({ description: "Quality по умолчанию для custom transform.", example: 80 }) + quality!: number +} + +export class PresetsResponseDto { + @ApiProperty({ description: "Static presets.", type: [PresetResponseDto] }) + presets!: PresetResponseDto[] + + @ApiProperty({ description: "Custom transform config.", type: CustomTransformConfigResponseDto }) + custom!: CustomTransformConfigResponseDto + + @ApiProperty({ description: "Mock allowlist source hosts.", example: ["storage.yandexcloud.net"] }) + allowedSourceHosts!: string[] +} diff --git a/apps/gateway/package.json b/apps/gateway/package.json index 74ae64b..ee9fc35 100644 --- a/apps/gateway/package.json +++ b/apps/gateway/package.json @@ -5,11 +5,12 @@ "type": "module", "scripts": { "build": "tsc -p tsconfig.build.json", - "dev": "tsx watch src/main.ts", - "start": "node dist/main.js", + "dev": "node --env-file-if-exists=../../.env ./node_modules/tsx/dist/cli.mjs watch src/main.ts", + "start": "node --env-file-if-exists=../../.env dist/main.js", "typecheck": "tsc --noEmit -p tsconfig.json" }, "dependencies": { + "@image-platform/image-config": "workspace:*", "fastify": "^5.8.5" }, "devDependencies": { diff --git a/apps/gateway/src/config.ts b/apps/gateway/src/config.ts index 5cad5e1..d8646c1 100644 --- a/apps/gateway/src/config.ts +++ b/apps/gateway/src/config.ts @@ -1,6 +1,8 @@ export type GatewayConfig = { backendUpstream: URL host: string + l1MaxEntries: number + l1TtlMs: number port: number } @@ -8,6 +10,8 @@ export function loadGatewayConfig(): GatewayConfig { return { backendUpstream: new URL(process.env.GATEWAY_BACKEND_UPSTREAM ?? "http://localhost:3001"), host: process.env.GATEWAY_HOST ?? "0.0.0.0", + l1MaxEntries: parsePositiveInteger(process.env.GATEWAY_L1_MAX_ENTRIES, 256), + l1TtlMs: parsePositiveInteger(process.env.GATEWAY_L1_TTL_MS, 10 * 60 * 1000), port: parsePort(process.env.GATEWAY_PORT, 8888), } } @@ -25,3 +29,13 @@ function parsePort(value: string | undefined, fallback: number) { return parsed } + +function parsePositiveInteger(value: string | undefined, fallback: number) { + if (!value) { + return fallback + } + + const parsed = Number.parseInt(value, 10) + + return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback +} diff --git a/apps/gateway/src/image-cache.ts b/apps/gateway/src/image-cache.ts new file mode 100644 index 0000000..12eba31 --- /dev/null +++ b/apps/gateway/src/image-cache.ts @@ -0,0 +1,52 @@ +export type CachedImage = { + body: Buffer + cacheControl: string + contentType: string + etag: string | null +} + +export class ImageMemoryCache { + private readonly entries = new Map() + + constructor( + private readonly maxEntries: number, + private readonly ttlMs: number, + ) {} + + get(key: string): CachedImage | null { + const entry = this.entries.get(key) + + if (!entry) { + return null + } + + if (entry.expiresAt <= Date.now()) { + this.entries.delete(key) + return null + } + + this.entries.delete(key) + this.entries.set(key, entry) + + return { + body: entry.body, + cacheControl: entry.cacheControl, + contentType: entry.contentType, + etag: entry.etag, + } + } + + set(key: string, image: CachedImage) { + this.entries.set(key, { ...image, expiresAt: Date.now() + this.ttlMs }) + + while (this.entries.size > this.maxEntries) { + const firstKey = this.entries.keys().next().value as string | undefined + + if (!firstKey) { + break + } + + this.entries.delete(firstKey) + } + } +} diff --git a/apps/gateway/src/server.ts b/apps/gateway/src/server.ts index e42235b..6f55f67 100644 --- a/apps/gateway/src/server.ts +++ b/apps/gateway/src/server.ts @@ -1,12 +1,23 @@ -import Fastify from "fastify" +import Fastify, { type FastifyReply } from "fastify" +import { + ImageTransformConfigError, + normalizeImageTransform, + parseBooleanFlag, + selectFormatForAccept, + type ActualImageFormat, + type ResizeMode, +} from "@image-platform/image-config" import type { GatewayConfig } from "./config.js" +import { ImageMemoryCache, type CachedImage } from "./image-cache.js" import { proxyToUpstream } from "./proxy.js" export function createGatewayServer(config: GatewayConfig) { const app = Fastify({ logger: true, }) + const imageCache = new ImageMemoryCache(config.l1MaxEntries, config.l1TtlMs) + const allowCustomTransforms = parseBooleanFlag(process.env.IMAGE_ALLOW_CUSTOM_TRANSFORMS, false) app.get("/health", async () => ({ service: "image-platform-gateway", @@ -26,19 +37,113 @@ export function createGatewayServer(config: GatewayConfig) { } const width = parseOptionalInteger(request.query.w) + const height = parseOptionalNonNegativeInteger(request.query.h) const quality = parseOptionalInteger(request.query.q) - const format = request.query.f ?? "auto" + const resize = parseResizeMode(request.query.fit) - return reply.code(501).header("cache-control", "no-store").send({ - assetId: request.params.assetId, - format, - message: "image gateway read-through pipeline is not implemented yet", + if ( + (request.query.w !== undefined && width === null) || + (request.query.h !== undefined && height === null) || + (request.query.q !== undefined && quality === null) + ) { + return reply.code(400).send({ + message: "w, h and q query params must be positive integers", + statusCode: 400, + }) + } + + if (request.query.fit !== undefined && resize === null) { + return reply.code(400).send({ + message: "fit query param must be fit or fill", + statusCode: 400, + }) + } + + const format = selectFormat({ + acceptHeader: request.headers.accept, + allowCustomTransforms, + preset: request.params.preset, + requestedFormat: request.query.f ?? "auto", + }) + + if (!format.ok) { + return reply.code(400).send({ + message: format.message, + statusCode: 400, + }) + } + + const transform = normalizeTransform({ + allowCustomTransforms, + format: format.value.format, + height, preset: request.params.preset, quality, - status: "not_implemented", - version, + requestedFormat: format.value.requestedFormat, + resize, width, }) + + if (!transform.ok) { + return reply.code(400).send({ + message: transform.message, + statusCode: 400, + }) + } + + const cacheKey = buildImageCacheKey({ + assetId: request.params.assetId, + format: transform.value.format, + height: transform.value.height, + preset: transform.value.preset, + quality: transform.value.quality, + resize: transform.value.resize, + version, + width: transform.value.width, + }) + const cached = imageCache.get(cacheKey) + const vary = transform.value.requestedFormat === "auto" ? "Accept" : null + + if (cached) { + return sendImage(reply, cached, "HIT", vary) + } + + const backendResponse = await fetch(new URL("/api/internal/images/ensure", config.backendUpstream), { + body: JSON.stringify({ + assetId: request.params.assetId, + format: transform.value.format, + height: transform.value.height, + preset: transform.value.preset, + quality: transform.value.quality, + requestedFormat: transform.value.requestedFormat, + resize: transform.value.resize, + version, + width: transform.value.width, + }), + headers: { "content-type": "application/json" }, + method: "POST", + }) + + if (!backendResponse.ok) { + const body = await backendResponse.text() + + return reply + .code(backendResponse.status) + .header("cache-control", "no-store") + .header("content-type", backendResponse.headers.get("content-type") ?? "text/plain; charset=utf-8") + .send(body) + } + + const image: CachedImage = { + body: Buffer.from(await backendResponse.arrayBuffer()), + cacheControl: backendResponse.headers.get("cache-control") ?? "public, max-age=31536000, immutable", + contentType: backendResponse.headers.get("content-type") ?? contentTypeForFormat(transform.value.format), + etag: backendResponse.headers.get("etag"), + } + + imageCache.set(cacheKey, image) + + return sendImage(reply, image, "MISS", vary) }, ) @@ -59,10 +164,23 @@ export function createGatewayServer(config: GatewayConfig) { type ImageQuery = { f?: string + fit?: string + h?: string q?: string w?: string } +type ImageCacheKeyInput = { + assetId: string + format: ActualImageFormat + height: number + preset: string + quality: number + resize: ResizeMode + version: number + width: number +} + type ImageParams = { assetId: string preset: string @@ -84,7 +202,96 @@ function parseOptionalInteger(value: string | undefined) { return null } + if (!/^\d+$/.test(value)) { + return null + } + const parsed = Number.parseInt(value, 10) - return Number.isFinite(parsed) ? parsed : null + return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : null +} + +function parseOptionalNonNegativeInteger(value: string | undefined) { + if (!value) { + return null + } + + if (!/^\d+$/.test(value)) { + return null + } + + const parsed = Number.parseInt(value, 10) + + return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : null +} + +function parseResizeMode(value: string | undefined): ResizeMode | null { + if (value === undefined) { + return null + } + + return value === "fit" || value === "fill" ? value : null +} + +function selectFormat(input: Parameters[0]) { + try { + return { ok: true as const, value: selectFormatForAccept(input) } + } catch (error) { + if (error instanceof ImageTransformConfigError) { + return { message: error.message, ok: false as const } + } + + throw error + } +} + +function buildImageCacheKey(input: ImageCacheKeyInput) { + return [ + input.assetId, + input.version, + input.preset, + input.width, + input.height, + input.resize, + input.quality, + input.format, + ].join(":") +} + +function normalizeTransform(input: Parameters[0]) { + try { + return { ok: true as const, value: normalizeImageTransform(input) } + } catch (error) { + if (error instanceof ImageTransformConfigError) { + return { message: error.message, ok: false as const } + } + + throw error + } +} + +function sendImage(reply: FastifyReply, image: CachedImage, cacheStatus: string, vary: string | null) { + if (image.etag) { + reply.header("etag", image.etag) + } + + if (vary) { + reply.header("vary", vary) + } + + return reply + .code(200) + .header("cache-control", image.cacheControl) + .header("content-length", image.body.length.toString()) + .header("content-type", image.contentType) + .header("x-image-platform-l1", cacheStatus) + .send(image.body) +} + +function contentTypeForFormat(format: ActualImageFormat) { + if (format === "jpg") { + return "image/jpeg" + } + + return `image/${format}` } diff --git a/apps/worker/package.json b/apps/worker/package.json index d155050..de38736 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -2,18 +2,19 @@ "name": "@image-platform/worker", "version": "0.1.0", "private": true, - "type": "module", "scripts": { "build": "tsc -p tsconfig.build.json", - "dev": "tsx watch src/main.ts", - "start": "node dist/main.js", + "dev": "node --env-file-if-exists=../../.env ./node_modules/tsx/dist/cli.mjs watch src/main.ts", + "start": "node --env-file-if-exists=../../.env dist/main.js", "typecheck": "tsc --noEmit -p tsconfig.json" }, "dependencies": { "@image-platform/database": "workspace:*", + "@image-platform/image-config": "workspace:*", "@image-platform/queue": "workspace:*", "@image-platform/storage": "workspace:*", - "amqplib": "^1.0.4" + "amqplib": "^1.0.4", + "drizzle-orm": "^0.45.2" }, "devDependencies": { "@types/amqplib": "^0.10.8", diff --git a/apps/worker/src/config.ts b/apps/worker/src/config.ts index c2598ca..414f008 100644 --- a/apps/worker/src/config.ts +++ b/apps/worker/src/config.ts @@ -1,19 +1,34 @@ import { loadQueueTopologyFromEnv, type QueueTopology } from "@image-platform/queue" +import { loadStorageConfigFromEnv, type StorageConfig } from "@image-platform/storage" export type WorkerConfig = { + imgproxyUpstream: URL prefetch: number queueTopology: QueueTopology rabbitmqUrl: string + storage: StorageConfig } export function loadWorkerConfig(env: NodeJS.ProcessEnv = process.env): WorkerConfig { return { + imgproxyUpstream: new URL(getRequiredEnv(env, "IMGPROXY_UPSTREAM")), prefetch: parsePositiveInteger(env.WORKER_PREFETCH, 2), queueTopology: loadQueueTopologyFromEnv(env), - rabbitmqUrl: env.RABBITMQ_URL ?? "amqp://image:image-password@localhost:5672/image_platform", + rabbitmqUrl: getRequiredEnv(env, "RABBITMQ_URL"), + storage: loadStorageConfigFromEnv(env), } } +function getRequiredEnv(env: NodeJS.ProcessEnv, name: string) { + const value = env[name] + + if (!value) { + throw new Error(`${name} is required`) + } + + return value +} + function parsePositiveInteger(value: string | undefined, fallback: number) { if (!value) { return fallback diff --git a/apps/worker/src/main.ts b/apps/worker/src/main.ts index 7d6e091..ecea7e6 100644 --- a/apps/worker/src/main.ts +++ b/apps/worker/src/main.ts @@ -1,10 +1,23 @@ import amqp, { type Channel, type ConsumeMessage } from "amqplib" -import { parseGenerateVariantJobBuffer, type QueueTopology } from "@image-platform/queue" +import { and, eq, sql } from "drizzle-orm" +import { + createDatabase, + createDatabasePool, + imageAssets, + imageAssetVersions, + imageVariants, +} from "@image-platform/database" +import { assertQueueTopology, parseGenerateVariantJobBuffer } from "@image-platform/queue" +import type { ResizeMode } from "@image-platform/image-config" +import { createS3Client, putObjectBuffer } from "@image-platform/storage" import { loadWorkerConfig } from "./config.js" async function bootstrap() { const config = loadWorkerConfig() + const pool = createDatabasePool() + const db = createDatabase(pool) + const s3 = createS3Client(config.storage) const connection = await amqp.connect(config.rabbitmqUrl) const channel = await connection.createChannel() @@ -12,7 +25,7 @@ async function bootstrap() { await channel.prefetch(config.prefetch) await channel.consume( config.queueTopology.generateVariantQueue, - (message) => void handleGenerateVariantMessage(channel, message), + (message) => void handleGenerateVariantMessage({ channel, config, db, message, s3 }), { noAck: false }, ) @@ -22,6 +35,7 @@ async function bootstrap() { console.log("worker shutting down") await channel.close().catch((error: unknown) => console.error("failed to close RabbitMQ channel", error)) await connection.close().catch((error: unknown) => console.error("failed to close RabbitMQ connection", error)) + await pool.end().catch((error: unknown) => console.error("failed to close PostgreSQL pool", error)) process.exit(0) } @@ -29,38 +43,179 @@ async function bootstrap() { process.once("SIGTERM", () => void shutdown()) } -async function assertQueueTopology(channel: Channel, topology: QueueTopology) { - await channel.assertExchange(topology.jobsExchange, "direct", { durable: true }) - await channel.assertExchange(topology.jobsDeadLetterExchange, "direct", { durable: true }) - await channel.assertQueue(topology.generateVariantQueue, { - deadLetterExchange: topology.jobsDeadLetterExchange, - deadLetterRoutingKey: topology.generateVariantDeadLetterRoutingKey, - durable: true, - }) - await channel.assertQueue(topology.generateVariantDeadLetterQueue, { durable: true }) - await channel.bindQueue(topology.generateVariantQueue, topology.jobsExchange, topology.generateVariantRoutingKey) - await channel.bindQueue( - topology.generateVariantDeadLetterQueue, - topology.jobsDeadLetterExchange, - topology.generateVariantDeadLetterRoutingKey, - ) +type GenerateVariantContext = { + channel: Channel + config: ReturnType + db: ReturnType + message: ConsumeMessage | null + s3: ReturnType } -async function handleGenerateVariantMessage(channel: Channel, message: ConsumeMessage | null) { +async function handleGenerateVariantMessage({ channel, config, db, message, s3 }: GenerateVariantContext) { if (message === null) { return } try { const job = parseGenerateVariantJobBuffer(message.content) - console.log("generate variant job received, handler not implemented yet", job) - channel.nack(message, false, false) + const variant = await loadVariantForGeneration(db, job.variantId) + + if (!variant) { + console.error("generate variant job references missing variant", job) + channel.nack(message, false, false) + return + } + + if (variant.status === "ready") { + channel.ack(message) + return + } + + await db + .update(imageVariants) + .set({ + attemptCount: sql`${imageVariants.attemptCount} + 1`, + error: null, + status: "processing", + updatedAt: new Date(), + }) + .where(and(eq(imageVariants.id, variant.id), eq(imageVariants.status, variant.status))) + + const imgproxyUrl = buildImgproxyUrl(config.imgproxyUpstream, variant.sourceUrl, { + format: variant.format, + height: variant.height ?? 0, + quality: variant.quality, + resize: variant.resizeMode, + width: variant.width, + }) + const response = await fetch(imgproxyUrl) + + if (!response.ok) { + throw new Error(`imgproxy returned ${response.status} for variant ${variant.id}`) + } + + const body = Buffer.from(await response.arrayBuffer()) + const contentType = response.headers.get("content-type") ?? contentTypeForFormat(variant.format) + const putResult = await putObjectBuffer({ + body, + bucket: config.storage.bucket, + cacheControl: "public, max-age=31536000, immutable", + client: s3, + contentType, + key: variant.s3Key, + }) + + await db + .update(imageVariants) + .set({ + contentType, + error: null, + etag: putResult.ETag ?? null, + sizeBytes: body.length, + status: "ready", + updatedAt: new Date(), + }) + .where(eq(imageVariants.id, variant.id)) + + console.log("generated image variant", { id: variant.id, key: variant.s3Key, sizeBytes: body.length }) + channel.ack(message) } catch (error) { console.error("invalid generate variant job", error) + const variantId = getVariantIdFromMessage(message) + + if (variantId) { + await db + .update(imageVariants) + .set({ error: formatError(error), status: "failed", updatedAt: new Date() }) + .where(eq(imageVariants.id, variantId)) + .catch((updateError: unknown) => console.error("failed to mark variant as failed", updateError)) + } + channel.nack(message, false, false) } } +async function loadVariantForGeneration(db: ReturnType, variantId: string) { + const [row] = await db + .select({ + format: imageVariants.format, + height: imageVariants.height, + id: imageVariants.id, + quality: imageVariants.quality, + resizeMode: imageVariants.resizeMode, + s3Key: imageVariants.s3Key, + sourceUrl: imageAssetVersions.sourceUrl, + status: imageVariants.status, + width: imageVariants.width, + }) + .from(imageVariants) + .innerJoin(imageAssets, eq(imageVariants.assetId, imageAssets.id)) + .innerJoin(imageAssetVersions, eq(imageVariants.assetVersionId, imageAssetVersions.id)) + .where(eq(imageVariants.id, variantId)) + .limit(1) + + return row ?? null +} + +function buildImgproxyUrl( + upstream: URL, + sourceUrl: string, + options: { + format: "avif" | "jpg" | "png" | "webp" + height: number + quality: number + resize: ResizeMode + width: number + }, +) { + const url = new URL(upstream) + const encodedSource = Buffer.from(sourceUrl).toString("base64url") + url.pathname = joinUrlPath( + url.pathname, + "insecure", + `rs:${options.resize}:${options.width}:${options.height}`, + `q:${options.quality}`, + `${encodedSource}.${options.format}`, + ) + + return url +} + +function joinUrlPath(...segments: string[]) { + return segments + .flatMap((segment) => segment.split("/")) + .filter(Boolean) + .map(encodePathSegment) + .join("/") + .replace(/^/, "/") +} + +function encodePathSegment(segment: string) { + return segment.includes(":") ? segment : encodeURIComponent(segment) +} + +function contentTypeForFormat(format: "avif" | "jpg" | "png" | "webp") { + if (format === "jpg") { + return "image/jpeg" + } + + return `image/${format}` +} + +function getVariantIdFromMessage(message: ConsumeMessage) { + try { + return parseGenerateVariantJobBuffer(message.content).variantId + } catch { + return null + } +} + +function formatError(error: unknown) { + const message = error instanceof Error ? error.message : String(error) + + return message.slice(0, 2000) +} + void bootstrap().catch((error: unknown) => { console.error("worker failed to start", error) process.exit(1) diff --git a/apps/worker/tsconfig.json b/apps/worker/tsconfig.json index 6bc598e..2583c30 100644 --- a/apps/worker/tsconfig.json +++ b/apps/worker/tsconfig.json @@ -5,8 +5,8 @@ "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "lib": ["ES2023"], - "module": "NodeNext", - "moduleResolution": "NodeNext", + "module": "Node16", + "moduleResolution": "Node16", "noUncheckedIndexedAccess": true, "outDir": "./dist", "rootDir": "./src", diff --git a/docs/backend-contract-draft.md b/docs/backend-contract-draft.md index 5e8a705..d6d0447 100644 --- a/docs/backend-contract-draft.md +++ b/docs/backend-contract-draft.md @@ -1,6 +1,6 @@ # Черновик Backend Контракта -Это черновик будущего бизнес-контракта для NestJS backend. Сейчас реализованы system endpoints, Swagger и placeholder для internal image ensure. +Это черновик бизнес-контракта для NestJS backend. Сейчас реализованы system endpoints, Swagger, `POST /api/assets` и internal image ensure MVP. ## System @@ -20,7 +20,7 @@ NestJS backend отдаёт JSON, metadata, statuses и URLs. Public image origi POST /api/internal/images/ensure ``` -Текущий статус реализации - endpoint зарезервирован и возвращает `501 Not Implemented`. Gateway `/images/*` пока тоже возвращает placeholder `501 Not Implemented`. +Текущий статус реализации - MVP реализован: Backend создаёт/переиспользует variant row, публикует RabbitMQ job, ждёт `ready` до timeout и возвращает image bytes из S3. Request body: @@ -30,7 +30,10 @@ Request body: "version": 4, "preset": "card", "width": 640, + "height": 0, "quality": 80, + "requestedFormat": "auto", + "resize": "fit", "format": "webp" } ``` @@ -39,13 +42,16 @@ Query params: | Param | Описание | |---|---| -| `w` | целевая ширина, должна быть разрешена preset или округлена до ближайшей разрешённой | -| `q` | качество, должно быть из allowlist качества | +| `w` | целевая ширина; обязательна для responsive presets и custom, запрещена/фиксирована для fixed presets | +| `h` | целевая высота; используется для custom, для fixed preset берётся из config | +| `q` | качество; если не передано, берётся из preset, иначе должно входить в allowlist preset | | `f` | `auto`, `avif`, `webp`, `jpg`; для Next/image по умолчанию `auto` | +| `fit` | `fit` или `fill`; используется только для custom transforms | Responsibilities: -- проверить `assetId` и `preset`; +- проверить `assetId` и статический preset/custom transform config; +- нормализовать width, height, resize, quality и format; - вычислить deterministic `variantHash`; - проверить PostgreSQL и S3; - если variant готов в S3, вернуть bytes или stream metadata для Gateway; @@ -69,9 +75,8 @@ ETag: "..." |---|---| | `400` | некорректные query params | | `404` | asset или preset не найден | -| `409` | variant уже генерируется и sync ожидание отключено | -| `422` | source image нельзя обработать | -| `502` | external imgproxy недоступен | +| `502` | external imgproxy недоступен или S3 object отсутствует после `ready` | +| `504` | generation не завершилась до `IMAGE_ENSURE_WAIT_MS` | Public endpoint реализуется в Fastify Gateway, internal ensure endpoint - в Backend: @@ -82,6 +87,14 @@ client -> CDN -> Gateway /images/* -> Backend ensure -> RabbitMQ -> Worker -> im Важно: `/images/*` не должен жить в NestJS `/api`. Gateway остаётся public image origin, Backend остаётся управлением процессов. Public image URL versioned: `/images/{assetId}/v{version}/{preset}?w={width}&q={quality}&f=auto`. +Примеры: + +```text +/images/asset_demo/v1/card?w=640&q=80&f=auto +/images/asset_demo/v1/avatar?f=auto +/images/asset_demo/v1/custom?w=777&h=333&q=72&fit=fill&f=webp +``` + ## Allowed Hosts ```text @@ -94,63 +107,152 @@ DELETE /allowed-hosts/:id ## Assets ```text -GET /assets POST /assets -GET /assets/:id +GET /assets +GET /assets/:publicId +GET /assets/:publicId/variants +POST /assets/:publicId/variants DELETE /assets/:id ``` +Сейчас реализованы `POST /assets`, `GET /assets`, `GET /assets/:publicId`, `GET /assets/:publicId/variants`, `POST /assets/:publicId/variants`. + `POST /assets` request: ```json { - "sourceUrl": "https://example.com/photo.jpg" + "sourceUrl": "https://storage.yandexcloud.net/shared1318/img/1.jpg", + "publicId": "asset_demo" +} +``` + +Response: + +```json +{ + "id": "59fcf4f6-9891-4df4-8bb7-a4dbe570bb66", + "publicId": "asset_demo", + "version": 1, + "sourceHost": "storage.yandexcloud.net", + "imageBasePath": "/images/asset_demo/v1/card" } ``` Responsibilities: - validate source URL; -- check `allowed_image_hosts`; -- create or reuse `image_assets` row; +- check mock allowlist `SOURCE_ALLOWED_HOSTS`, если `SOURCE_HOST_ALLOW_ALL=false`; +- создать `image_assets` row; +- создать `image_asset_versions` row версии `1`; - optionally save original to S3 later. +`GET /assets` response: + +```json +{ + "assets": [ + { + "id": "59fcf4f6-9891-4df4-8bb7-a4dbe570bb66", + "publicId": "asset_demo", + "currentVersion": 1, + "status": "active", + "sourceUrl": "https://storage.yandexcloud.net/shared1318/img/1.jpg", + "sourceHost": "storage.yandexcloud.net", + "createdAt": "2026-05-05T12:00:00.000Z", + "updatedAt": "2026-05-05T12:00:00.000Z" + } + ] +} +``` + +`GET /assets/:publicId/variants` возвращает rows из `image_variants` с public Gateway URL, S3 key и status. + ## Variants ```text -GET /assets/:id/variants -POST /assets/:id/variants +GET /assets/:publicId/variants +POST /assets/:publicId/variants POST /variants/:id/regenerate DELETE /variants/:id ``` -`POST /assets/:id/variants` request: +`POST /assets/:publicId/variants` request: ```json { "preset": "card", + "mode": "single", "format": "webp", - "width": 640 + "width": 640, + "quality": 80 } ``` -Response if ready: +Family generation: ```json { - "id": "variant_123", - "status": "ready", - "url": "http://localhost:8888/images/asset_123/v4/card?w=640&q=80&f=webp" + "preset": "card", + "mode": "family" } ``` -Response if generation is async: +Для `family` Backend создаёт набор variants по разрешённым widths/formats static preset и ставит RabbitMQ jobs. Endpoint не ждёт image bytes, в отличие от internal ensure. + +Response: ```json { - "id": "variant_123", - "status": "pending", - "url": null + "publicId": "asset_demo", + "version": 1, + "variants": [ + { + "id": "variant_123", + "preset": "card", + "version": 1, + "width": 640, + "height": 0, + "resize": "fit", + "quality": 80, + "requestedFormat": "webp", + "format": "webp", + "status": "pending", + "url": "http://localhost:8888/images/asset_demo/v1/card?w=640&q=80&f=webp", + "s3Key": "variants/asset_demo/v1/abc.webp" + } + ] +} +``` + +## Presets + +```text +GET /presets +``` + +Возвращает static presets, custom transform limits и mock allowlist source hosts. + +```json +{ + "presets": [ + { + "name": "card", + "mode": "responsive", + "formats": ["avif", "webp", "jpg"], + "qualities": [75, 80], + "quality": 80, + "resize": "fit", + "widths": [320, 640, 960] + } + ], + "custom": { + "enabled": true, + "formats": ["avif", "webp", "jpg", "png"], + "maxWidth": 4096, + "maxHeight": 4096, + "quality": 80 + }, + "allowedSourceHosts": ["storage.yandexcloud.net"] } ``` diff --git a/docs/data-model.md b/docs/data-model.md index 6c268cf..94f9eb6 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -68,6 +68,7 @@ requested_format requested_format not null default auto format variant_format not null width integer not null height integer nullable +resize_mode resize_mode not null default fit quality integer not null s3_key text not null unique content_type text nullable @@ -89,6 +90,7 @@ updated_at timestamptz not null default now() asset_status: active | disabled | deleted requested_format: auto | avif | webp | jpg | png variant_format: avif | webp | jpg | png +resize_mode: fit | fill variant_status: pending | processing | ready | failed ``` @@ -98,7 +100,7 @@ variant_status: pending | processing | ready | failed allowed_image_hosts(hostname) image_assets(public_id) image_asset_versions(asset_id, version) -image_variants(asset_id, asset_version, preset, width, quality, format) +image_variants(asset_id, asset_version, preset, width, height, resize_mode, quality, format) image_variants(s3_key) image_variants(variant_hash) ``` @@ -123,6 +125,8 @@ variants/{assetId}/v{version}/{variantHash}.{format} - `assetVersion`; - `preset`; - normalized width; +- normalized height, где `0` означает auto height; +- normalized resize mode; - normalized quality; - фактический output format; - параметры transform, влияющие на bytes. @@ -135,34 +139,45 @@ variants/asset_123/v4/card_w640_q80_webp.webp variants/asset_123/v4/card_w640_q80_jpg.jpg ``` -Public URL также versioned: +Public URL также versioned. Для fixed preset `w` и `q` можно не передавать, для responsive preset `w` обязателен: ```text /images/{assetId}/v{version}/{preset}?w={width}&q={quality}&f=auto +/images/{assetId}/v{version}/avatar?f=auto ``` ## Presets -Клиент не должен передавать произвольные трансформации. Сначала нужны ограниченные presets. +Клиент не должен бесконтрольно создавать произвольные трансформации. Сейчас есть статический config в `packages/image-config`. + +Режимы: + +- `fixed` - preset задаёт один размер, например `avatar`. +- `responsive` - preset задаёт allowlist ширин, например `card` и `hero`. +- `custom` - произвольный single image, только если включён `IMAGE_ALLOW_CUSTOM_TRANSFORMS=true`. Пример: ```text avatar: - widths: 128, 256, 512 + mode: fixed + width: 256 + height: 256 formats: avif, webp, jpg quality: 80 resize: fill card: + mode: responsive widths: 320, 640, 960 formats: avif, webp, jpg - quality: 80 + qualities: 75, 80 resize: fit hero: + mode: responsive widths: 1280, 1920 formats: avif, webp, jpg - quality: 80 + qualities: 75, 80 resize: fit ``` diff --git a/docs/development.md b/docs/development.md index bdc7d0f..4bde4af 100644 --- a/docs/development.md +++ b/docs/development.md @@ -12,7 +12,7 @@ - imgproxy dev instance; - RabbitMQ. -`backend` уже создан как NestJS app. `admin` уже создан как чистый Vite React TS app. `gateway` уже создан как Fastify app. `worker` уже создан как RabbitMQ skeleton. +`backend` уже умеет регистрировать assets и выполнять internal ensure. `gateway` уже ходит в Backend и держит L1 memory cache. `worker` уже читает RabbitMQ jobs, вызывает imgproxy и пишет variants в S3. Gateway обязателен для Cloudinary-like поведения и интеграции с `next/image`. ## Запуск инфраструктуры @@ -21,8 +21,11 @@ Gateway обязателен для Cloudinary-like поведения и инт cp .env.example .env pnpm install pnpm infra:up +pnpm db:migrate ``` +`.env` используется только локально и игнорируется git. Backend, Gateway, Worker и Drizzle scripts автоматически подхватывают его через Node `--env-file-if-exists`; в production эти переменные должны приходить из окружения процесса. + Проверить compose config: ```bash @@ -73,6 +76,31 @@ curl http://localhost:3001/api/health open http://localhost:3001/docs ``` +Зарегистрировать source image в dev mode: + +```bash +curl -sS -X POST http://localhost:3001/api/assets \ + -H 'content-type: application/json' \ + -d '{"sourceUrl":"https://storage.yandexcloud.net/shared1318/img/1.jpg","publicId":"asset_demo"}' +``` + +Посмотреть business API: + +```bash +curl -sS http://localhost:3001/api/presets +curl -sS http://localhost:3001/api/assets +curl -sS http://localhost:3001/api/assets/asset_demo +curl -sS http://localhost:3001/api/assets/asset_demo/variants +``` + +Явно поставить jobs на генерацию family variants без Gateway lazy request: + +```bash +curl -sS -X POST http://localhost:3001/api/assets/asset_demo/variants \ + -H 'content-type: application/json' \ + -d '{"preset":"card","mode":"family"}' +``` + ## Database Drizzle schema живёт в `packages/database/src/schema.ts`, миграции - в `packages/database/drizzle`. @@ -130,17 +158,27 @@ pnpm gateway:dev curl http://localhost:8888/health ``` -Проверить placeholder image origin route: +Проверить image origin route после запуска Backend и Worker: ```bash -curl -i "http://localhost:8888/images/asset_123/v4/card?w=640&q=80&f=auto" +curl -i "http://localhost:8888/images/asset_demo/v1/card?w=640&q=80&f=auto" +curl -i "http://localhost:8888/images/asset_demo/v1/avatar?f=auto" +curl -i "http://localhost:8888/images/asset_demo/v1/custom?w=777&h=333&q=72&fit=fill&f=webp" ``` -Ожидаемый текущий статус - `501`, потому что S3/imgproxy генерация ещё не реализована. +Первый запрос должен пройти через Backend/RabbitMQ/Worker/imgproxy/S3 и вернуться с `x-image-platform-l1: MISS`. Повторный запрос должен вернуться из gateway L1 с `x-image-platform-l1: HIT`. + +Статические presets сейчас лежат в `packages/image-config`: + +- `card` - responsive, widths `320`, `640`, `960`. +- `hero` - responsive, widths `1280`, `1920`. +- `avatar` - fixed `256x256`. + +Mock allowlist source hosts задаётся через `SOURCE_ALLOWED_HOSTS`. В dev по умолчанию разрешён `storage.yandexcloud.net`. ## Worker -Worker запускается нодой и сейчас только объявляет RabbitMQ topology, слушает `image.generate-variant` и отправляет полученные jobs в DLQ, потому что генерация ещё не реализована. +Worker запускается нодой, объявляет RabbitMQ topology, слушает `image.generate-variant`, вызывает `imgproxy` и пишет готовый variant в S3. ```bash pnpm worker:dev diff --git a/docs/imgproxy-contract.md b/docs/imgproxy-contract.md index 2b573a9..cddd965 100644 --- a/docs/imgproxy-contract.md +++ b/docs/imgproxy-contract.md @@ -18,7 +18,7 @@ IMGPROXY_SALT= Пример path: ```text -/unsafe/resize:fit:800:0:0/q:80/plain/https://example.com/photo.jpg +/unsafe/resize:fit:800:0:0/q:80/plain/https://storage.yandexcloud.net/shared1318/img/1.jpg ``` ## Prod режим @@ -28,7 +28,7 @@ IMGPROXY_SALT= Path для подписи строится без `/unsafe`: ```text -/resize:fit:800:0:0/q:80/plain/https://example.com/photo.jpg +/resize:fit:800:0:0/q:80/plain/https://storage.yandexcloud.net/shared1318/img/1.jpg ``` Signature: @@ -66,6 +66,6 @@ Final signed URL: - Source URL валидировать в Backend/worker. - Разрешать только `http` и `https`. - Запрещать localhost, private IP, loopback, link-local. -- Source host должен быть enabled в `allowed_image_hosts`. -- Не давать клиенту произвольные imgproxy options. -- Использовать presets и deterministic `variantHash`. +- Source host должен быть разрешён mock allowlist `SOURCE_ALLOWED_HOSTS`; таблица `allowed_image_hosts` остаётся для будущего CRUD. +- Не давать клиенту произвольные imgproxy options без `IMAGE_ALLOW_CUSTOM_TRANSFORMS=true`. +- Использовать static presets/custom normalization и deterministic `variantHash`. diff --git a/docs/next-image-provider.md b/docs/next-image-provider.md index 8589cc2..2f5e01b 100644 --- a/docs/next-image-provider.md +++ b/docs/next-image-provider.md @@ -22,7 +22,7 @@ module.exports = { images: { loader: "custom", loaderFile: "./src/image-platform-loader.js", - qualities: [60, 75, 80, 90], + qualities: [75, 80], }, } ``` @@ -58,7 +58,7 @@ export function ProductCard() { GET /images/{assetId}/v{version}/{preset}?w={width}&q={quality}&f=auto ``` -Сейчас route уже зарезервирован в Fastify Gateway, но возвращает placeholder `501`, пока не реализованы PostgreSQL/S3/imgproxy read-through операции. +Route реализован в Fastify Gateway. Для `card` ширина должна входить в static preset allowlist: `320`, `640`, `960`. Пример: diff --git a/package.json b/package.json index 942048d..f9f72df 100644 --- a/package.json +++ b/package.json @@ -13,31 +13,33 @@ "admin:dev": "pnpm --filter @image-platform/admin dev", "admin:preview": "pnpm --filter @image-platform/admin preview", "admin:typecheck": "pnpm --filter @image-platform/admin typecheck", - "backend:build": "pnpm --filter @image-platform/backend build", - "backend:dev": "pnpm --filter @image-platform/backend dev", - "backend:start": "pnpm --filter @image-platform/backend start", + "backend:build": "pnpm image-config:build && pnpm db:build && pnpm queue:build && pnpm storage:build && pnpm --filter @image-platform/backend build", + "backend:dev": "pnpm image-config:build && pnpm db:build && pnpm queue:build && pnpm storage:build && pnpm --filter @image-platform/backend dev", + "backend:start": "pnpm image-config:build && pnpm db:build && pnpm queue:build && pnpm storage:build && pnpm --filter @image-platform/backend start", "backend:typecheck": "pnpm --filter @image-platform/backend typecheck", "db:build": "pnpm --filter @image-platform/database build", "db:generate": "pnpm --filter @image-platform/database db:generate", "db:migrate": "pnpm --filter @image-platform/database db:migrate", "db:studio": "pnpm --filter @image-platform/database db:studio", "db:typecheck": "pnpm --filter @image-platform/database typecheck", - "gateway:build": "pnpm --filter @image-platform/gateway build", - "gateway:dev": "pnpm --filter @image-platform/gateway dev", - "gateway:start": "pnpm --filter @image-platform/gateway start", + "gateway:build": "pnpm image-config:build && pnpm --filter @image-platform/gateway build", + "gateway:dev": "pnpm image-config:build && pnpm --filter @image-platform/gateway dev", + "gateway:start": "pnpm image-config:build && pnpm --filter @image-platform/gateway start", "gateway:typecheck": "pnpm --filter @image-platform/gateway typecheck", + "image-config:build": "pnpm --filter @image-platform/image-config build", + "image-config:typecheck": "pnpm --filter @image-platform/image-config typecheck", "queue:build": "pnpm --filter @image-platform/queue build", "queue:typecheck": "pnpm --filter @image-platform/queue typecheck", "storage:build": "pnpm --filter @image-platform/storage build", "storage:typecheck": "pnpm --filter @image-platform/storage typecheck", - "worker:build": "pnpm db:build && pnpm queue:build && pnpm storage:build && pnpm --filter @image-platform/worker build", - "worker:dev": "pnpm db:build && pnpm queue:build && pnpm storage:build && pnpm --filter @image-platform/worker dev", - "worker:start": "pnpm --filter @image-platform/worker start", + "worker:build": "pnpm image-config:build && pnpm db:build && pnpm queue:build && pnpm storage:build && pnpm --filter @image-platform/worker build", + "worker:dev": "pnpm image-config:build && pnpm db:build && pnpm queue:build && pnpm storage:build && pnpm --filter @image-platform/worker dev", + "worker:start": "pnpm image-config:build && pnpm --filter @image-platform/worker start", "worker:typecheck": "pnpm --filter @image-platform/worker typecheck", "infra:config": "docker compose -f infra/compose.dev.yml config", "infra:up": "docker compose -f infra/compose.dev.yml up -d", "infra:down": "docker compose -f infra/compose.dev.yml down", "infra:logs": "docker compose -f infra/compose.dev.yml logs -f", - "check": "pnpm infra:config && pnpm db:typecheck && pnpm queue:typecheck && pnpm storage:typecheck && pnpm backend:typecheck && pnpm admin:typecheck && pnpm gateway:typecheck && pnpm worker:typecheck" + "check": "pnpm infra:config && pnpm image-config:typecheck && pnpm db:typecheck && pnpm queue:typecheck && pnpm storage:typecheck && pnpm backend:typecheck && pnpm admin:typecheck && pnpm gateway:typecheck && pnpm worker:typecheck" } } diff --git a/packages/database/drizzle.config.ts b/packages/database/drizzle.config.ts index 6c93611..e8b30cb 100644 --- a/packages/database/drizzle.config.ts +++ b/packages/database/drizzle.config.ts @@ -1,8 +1,14 @@ import { defineConfig } from "drizzle-kit" +const databaseUrl = process.env.DATABASE_URL + +if (!databaseUrl) { + throw new Error("DATABASE_URL is required") +} + export default defineConfig({ dbCredentials: { - url: process.env.DATABASE_URL ?? "postgres://image:image-password@localhost:5433/image_platform", + url: databaseUrl, }, dialect: "postgresql", out: "./drizzle", diff --git a/packages/database/drizzle/0001_familiar_nextwave.sql b/packages/database/drizzle/0001_familiar_nextwave.sql new file mode 100644 index 0000000..489b0e9 --- /dev/null +++ b/packages/database/drizzle/0001_familiar_nextwave.sql @@ -0,0 +1,4 @@ +CREATE TYPE "public"."resize_mode" AS ENUM('fit', 'fill');--> statement-breakpoint +DROP INDEX "image_variants_lookup_idx";--> statement-breakpoint +ALTER TABLE "image_variants" ADD COLUMN "resize_mode" "resize_mode" DEFAULT 'fit' NOT NULL;--> statement-breakpoint +CREATE UNIQUE INDEX "image_variants_lookup_idx" ON "image_variants" USING btree ("asset_id","asset_version","preset","width","height","resize_mode","quality","format"); \ No newline at end of file diff --git a/packages/database/drizzle/meta/0001_snapshot.json b/packages/database/drizzle/meta/0001_snapshot.json new file mode 100644 index 0000000..e485e51 --- /dev/null +++ b/packages/database/drizzle/meta/0001_snapshot.json @@ -0,0 +1,632 @@ +{ + "id": "9b706710-b809-4324-8632-634884f75166", + "prevId": "72292622-d326-46fe-8e6a-90096c7e6634", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.allowed_image_hosts": { + "name": "allowed_image_hosts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "hostname": { + "name": "hostname", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "allowed_image_hosts_hostname_idx": { + "name": "allowed_image_hosts_hostname_idx", + "columns": [ + { + "expression": "hostname", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.image_asset_versions": { + "name": "image_asset_versions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "asset_id": { + "name": "asset_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_host": { + "name": "source_host", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_hash": { + "name": "source_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "original_s3_key": { + "name": "original_s3_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "image_asset_versions_asset_version_idx": { + "name": "image_asset_versions_asset_version_idx", + "columns": [ + { + "expression": "asset_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "image_asset_versions_source_hash_idx": { + "name": "image_asset_versions_source_hash_idx", + "columns": [ + { + "expression": "source_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "image_asset_versions_asset_id_image_assets_id_fk": { + "name": "image_asset_versions_asset_id_image_assets_id_fk", + "tableFrom": "image_asset_versions", + "tableTo": "image_assets", + "columnsFrom": [ + "asset_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.image_assets": { + "name": "image_assets", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "public_id": { + "name": "public_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "current_version": { + "name": "current_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "status": { + "name": "status", + "type": "asset_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "image_assets_public_id_idx": { + "name": "image_assets_public_id_idx", + "columns": [ + { + "expression": "public_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.image_variants": { + "name": "image_variants", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "asset_id": { + "name": "asset_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "asset_version_id": { + "name": "asset_version_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "asset_version": { + "name": "asset_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "preset": { + "name": "preset", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "variant_hash": { + "name": "variant_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "requested_format": { + "name": "requested_format", + "type": "requested_format", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'auto'" + }, + "format": { + "name": "format", + "type": "variant_format", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "resize_mode": { + "name": "resize_mode", + "type": "resize_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'fit'" + }, + "quality": { + "name": "quality", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "s3_key": { + "name": "s3_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "etag": { + "name": "etag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "variant_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_accessed_at": { + "name": "last_accessed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "image_variants_lookup_idx": { + "name": "image_variants_lookup_idx", + "columns": [ + { + "expression": "asset_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "asset_version", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "preset", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "width", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resize_mode", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "quality", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "format", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "image_variants_s3_key_idx": { + "name": "image_variants_s3_key_idx", + "columns": [ + { + "expression": "s3_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "image_variants_variant_hash_idx": { + "name": "image_variants_variant_hash_idx", + "columns": [ + { + "expression": "variant_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "image_variants_status_idx": { + "name": "image_variants_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "image_variants_asset_id_image_assets_id_fk": { + "name": "image_variants_asset_id_image_assets_id_fk", + "tableFrom": "image_variants", + "tableTo": "image_assets", + "columnsFrom": [ + "asset_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "image_variants_asset_version_id_image_asset_versions_id_fk": { + "name": "image_variants_asset_version_id_image_asset_versions_id_fk", + "tableFrom": "image_variants", + "tableTo": "image_asset_versions", + "columnsFrom": [ + "asset_version_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.asset_status": { + "name": "asset_status", + "schema": "public", + "values": [ + "active", + "disabled", + "deleted" + ] + }, + "public.requested_format": { + "name": "requested_format", + "schema": "public", + "values": [ + "auto", + "avif", + "webp", + "jpg", + "png" + ] + }, + "public.resize_mode": { + "name": "resize_mode", + "schema": "public", + "values": [ + "fit", + "fill" + ] + }, + "public.variant_format": { + "name": "variant_format", + "schema": "public", + "values": [ + "avif", + "webp", + "jpg", + "png" + ] + }, + "public.variant_status": { + "name": "variant_status", + "schema": "public", + "values": [ + "pending", + "processing", + "ready", + "failed" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/database/drizzle/meta/_journal.json b/packages/database/drizzle/meta/_journal.json index dfeb3fe..996df66 100644 --- a/packages/database/drizzle/meta/_journal.json +++ b/packages/database/drizzle/meta/_journal.json @@ -8,6 +8,13 @@ "when": 1777963363578, "tag": "0000_calm_magdalene", "breakpoints": true + }, + { + "idx": 1, + "version": "7", + "when": 1777973330318, + "tag": "0001_familiar_nextwave", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/database/package.json b/packages/database/package.json index 3a0998d..75f01b8 100644 --- a/packages/database/package.json +++ b/packages/database/package.json @@ -2,19 +2,20 @@ "name": "@image-platform/database", "version": "0.1.0", "private": true, - "type": "module", "exports": { ".": { "types": "./src/index.ts", + "require": "./dist/index.js", + "import": "./dist/index.js", "default": "./dist/index.js" } }, "types": "./src/index.ts", "scripts": { "build": "tsc -p tsconfig.build.json", - "db:generate": "drizzle-kit generate --config drizzle.config.ts", - "db:migrate": "drizzle-kit migrate --config drizzle.config.ts", - "db:studio": "drizzle-kit studio --config drizzle.config.ts", + "db:generate": "node --env-file-if-exists=../../.env ./node_modules/drizzle-kit/bin.cjs generate --config drizzle.config.ts", + "db:migrate": "node --env-file-if-exists=../../.env ./node_modules/drizzle-kit/bin.cjs migrate --config drizzle.config.ts", + "db:studio": "node --env-file-if-exists=../../.env ./node_modules/drizzle-kit/bin.cjs studio --config drizzle.config.ts", "typecheck": "tsc --noEmit -p tsconfig.json" }, "dependencies": { diff --git a/packages/database/src/index.ts b/packages/database/src/index.ts index 91179c5..a4154ad 100644 --- a/packages/database/src/index.ts +++ b/packages/database/src/index.ts @@ -1,2 +1,3 @@ export { createDatabase, createDatabasePool } from "./client.js" +export type { Database, DatabasePool } from "./client.js" export * from "./schema.js" diff --git a/packages/database/src/schema.ts b/packages/database/src/schema.ts index eb19c90..a3d6141 100644 --- a/packages/database/src/schema.ts +++ b/packages/database/src/schema.ts @@ -14,6 +14,7 @@ import { export const assetStatusEnum = pgEnum("asset_status", ["active", "disabled", "deleted"]) export const variantFormatEnum = pgEnum("variant_format", ["avif", "webp", "jpg", "png"]) export const requestedFormatEnum = pgEnum("requested_format", ["auto", "avif", "webp", "jpg", "png"]) +export const resizeModeEnum = pgEnum("resize_mode", ["fit", "fill"]) export const variantStatusEnum = pgEnum("variant_status", ["pending", "processing", "ready", "failed"]) const timestamps = { @@ -86,6 +87,7 @@ export const imageVariants = pgTable( format: variantFormatEnum("format").notNull(), width: integer("width").notNull(), height: integer("height"), + resizeMode: resizeModeEnum("resize_mode").notNull().default("fit"), quality: integer("quality").notNull(), s3Key: text("s3_key").notNull(), contentType: text("content_type"), @@ -103,6 +105,8 @@ export const imageVariants = pgTable( table.assetVersion, table.preset, table.width, + table.height, + table.resizeMode, table.quality, table.format, ), diff --git a/packages/database/tsconfig.json b/packages/database/tsconfig.json index 6bc598e..2583c30 100644 --- a/packages/database/tsconfig.json +++ b/packages/database/tsconfig.json @@ -5,8 +5,8 @@ "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "lib": ["ES2023"], - "module": "NodeNext", - "moduleResolution": "NodeNext", + "module": "Node16", + "moduleResolution": "Node16", "noUncheckedIndexedAccess": true, "outDir": "./dist", "rootDir": "./src", diff --git a/packages/image-config/package.json b/packages/image-config/package.json new file mode 100644 index 0000000..d969bf7 --- /dev/null +++ b/packages/image-config/package.json @@ -0,0 +1,22 @@ +{ + "name": "@image-platform/image-config", + "version": "0.1.0", + "private": true, + "exports": { + ".": { + "types": "./src/index.ts", + "require": "./dist/index.js", + "import": "./dist/index.js", + "default": "./dist/index.js" + } + }, + "types": "./src/index.ts", + "scripts": { + "build": "tsc -p tsconfig.build.json", + "typecheck": "tsc --noEmit -p tsconfig.json" + }, + "devDependencies": { + "@types/node": "^25.6.0", + "typescript": "^6.0.3" + } +} diff --git a/packages/image-config/src/allowed-hosts.ts b/packages/image-config/src/allowed-hosts.ts new file mode 100644 index 0000000..1396e99 --- /dev/null +++ b/packages/image-config/src/allowed-hosts.ts @@ -0,0 +1,20 @@ +export const DEFAULT_ALLOWED_SOURCE_HOSTS = ["storage.yandexcloud.net"] as const + +export function loadAllowedSourceHostsFromEnv(env: NodeJS.ProcessEnv = process.env) { + const value = env.SOURCE_ALLOWED_HOSTS + + if (!value) { + return new Set(DEFAULT_ALLOWED_SOURCE_HOSTS) + } + + return new Set( + value + .split(",") + .map((host) => host.trim().toLowerCase()) + .filter(Boolean), + ) +} + +export function isAllowedSourceHost(hostname: string, allowedHosts: ReadonlySet) { + return allowedHosts.has(hostname.toLowerCase()) +} diff --git a/packages/image-config/src/index.ts b/packages/image-config/src/index.ts new file mode 100644 index 0000000..ff7bab9 --- /dev/null +++ b/packages/image-config/src/index.ts @@ -0,0 +1,2 @@ +export * from "./allowed-hosts.js" +export * from "./presets.js" diff --git a/packages/image-config/src/presets.ts b/packages/image-config/src/presets.ts new file mode 100644 index 0000000..30fdcf5 --- /dev/null +++ b/packages/image-config/src/presets.ts @@ -0,0 +1,297 @@ +export type ActualImageFormat = "avif" | "jpg" | "png" | "webp" +export type RequestedImageFormat = "auto" | ActualImageFormat +export type ResizeMode = "fill" | "fit" +export type PresetMode = "fixed" | "responsive" +export type TransformMode = "custom" | PresetMode + +export type ImagePreset = { + formats: readonly ActualImageFormat[] + height?: number + mode: PresetMode + qualities: readonly number[] + quality: number + resize: ResizeMode + width?: number + widths?: readonly number[] +} + +export type CustomTransformConfig = { + formats: readonly ActualImageFormat[] + maxHeight: number + maxWidth: number + quality: number +} + +export type NormalizeImageTransformInput = { + allowCustomTransforms: boolean + format: ActualImageFormat + height?: number | null + preset: string + quality?: number | null + requestedFormat?: RequestedImageFormat | null + resize?: ResizeMode | null + width?: number | null +} + +export type NormalizedImageTransform = { + format: ActualImageFormat + height: number + mode: TransformMode + preset: string + quality: number + requestedFormat: RequestedImageFormat + resize: ResizeMode + width: number +} + +export class ImageTransformConfigError extends Error { + constructor(message: string) { + super(message) + this.name = "ImageTransformConfigError" + } +} + +export const IMAGE_PRESETS = { + avatar: { + formats: ["avif", "webp", "jpg"], + height: 256, + mode: "fixed", + qualities: [80], + quality: 80, + resize: "fill", + width: 256, + }, + card: { + formats: ["avif", "webp", "jpg"], + mode: "responsive", + qualities: [75, 80], + quality: 80, + resize: "fit", + widths: [320, 640, 960], + }, + hero: { + formats: ["avif", "webp", "jpg"], + mode: "responsive", + qualities: [75, 80], + quality: 80, + resize: "fit", + widths: [1280, 1920], + }, +} as const satisfies Record + +export const CUSTOM_PRESET_NAME = "custom" + +export const CUSTOM_TRANSFORM_CONFIG: CustomTransformConfig = { + formats: ["avif", "webp", "jpg", "png"], + maxHeight: 4096, + maxWidth: 4096, + quality: 80, +} + +export function getImagePreset(name: string): ImagePreset | null { + return Object.hasOwn(IMAGE_PRESETS, name) ? IMAGE_PRESETS[name as keyof typeof IMAGE_PRESETS] : null +} + +export function normalizeImageTransform(input: NormalizeImageTransformInput): NormalizedImageTransform { + const requestedFormat = input.requestedFormat ?? input.format + const preset = getImagePreset(input.preset) + + if (!isRequestedImageFormat(requestedFormat)) { + throw new ImageTransformConfigError("requestedFormat is invalid") + } + + if (!isActualImageFormat(input.format)) { + throw new ImageTransformConfigError("format is invalid") + } + + if (preset) { + return normalizePresetTransform(input, preset, requestedFormat) + } + + if (input.preset === CUSTOM_PRESET_NAME) { + return normalizeCustomTransform(input, requestedFormat) + } + + throw new ImageTransformConfigError(`unknown image preset: ${input.preset}`) +} + +export function selectFormatForAccept(input: { + allowCustomTransforms: boolean + acceptHeader?: string | string[] + preset: string + requestedFormat: string +}): { format: ActualImageFormat; requestedFormat: RequestedImageFormat } { + if (!isRequestedImageFormat(input.requestedFormat)) { + throw new ImageTransformConfigError("requested format is invalid") + } + + const formats = getAllowedFormats(input.preset, input.allowCustomTransforms) + + if (input.requestedFormat !== "auto") { + if (!formats.includes(input.requestedFormat)) { + throw new ImageTransformConfigError(`format ${input.requestedFormat} is not allowed for ${input.preset}`) + } + + return { format: input.requestedFormat, requestedFormat: input.requestedFormat } + } + + const accept = Array.isArray(input.acceptHeader) ? input.acceptHeader.join(",") : (input.acceptHeader ?? "") + + if (accept.includes("image/avif") && formats.includes("avif")) { + return { format: "avif", requestedFormat: "auto" } + } + + if (accept.includes("image/webp") && formats.includes("webp")) { + return { format: "webp", requestedFormat: "auto" } + } + + if (formats.includes("jpg")) { + return { format: "jpg", requestedFormat: "auto" } + } + + if (formats.includes("png")) { + return { format: "png", requestedFormat: "auto" } + } + + throw new ImageTransformConfigError(`no fallback format configured for ${input.preset}`) +} + +export function parseBooleanFlag(value: string | undefined, fallback: boolean) { + if (value === undefined) { + return fallback + } + + return ["1", "true", "yes"].includes(value.toLowerCase()) +} + +function normalizePresetTransform( + input: NormalizeImageTransformInput, + preset: ImagePreset, + requestedFormat: RequestedImageFormat, +): NormalizedImageTransform { + if (!preset.formats.includes(input.format)) { + throw new ImageTransformConfigError(`format ${input.format} is not allowed for preset ${input.preset}`) + } + + const quality = normalizePresetQuality(input.quality, preset) + + if (preset.mode === "fixed") { + if (!preset.width || !preset.height) { + throw new ImageTransformConfigError(`fixed preset ${input.preset} must define width and height`) + } + + if (input.width !== null && input.width !== undefined && input.width !== preset.width) { + throw new ImageTransformConfigError(`width must be ${preset.width} for preset ${input.preset}`) + } + + if (input.height !== null && input.height !== undefined && input.height !== preset.height) { + throw new ImageTransformConfigError(`height must be ${preset.height} for preset ${input.preset}`) + } + + return { + format: input.format, + height: preset.height, + mode: preset.mode, + preset: input.preset, + quality, + requestedFormat, + resize: preset.resize, + width: preset.width, + } + } + + if (!input.width) { + throw new ImageTransformConfigError(`width is required for responsive preset ${input.preset}`) + } + + if (!preset.widths?.includes(input.width)) { + throw new ImageTransformConfigError(`width ${input.width} is not allowed for preset ${input.preset}`) + } + + if (input.height !== null && input.height !== undefined && input.height !== (preset.height ?? 0)) { + throw new ImageTransformConfigError(`height is not configurable for preset ${input.preset}`) + } + + return { + format: input.format, + height: preset.height ?? 0, + mode: preset.mode, + preset: input.preset, + quality, + requestedFormat, + resize: preset.resize, + width: input.width, + } +} + +function normalizeCustomTransform( + input: NormalizeImageTransformInput, + requestedFormat: RequestedImageFormat, +): NormalizedImageTransform { + if (!input.allowCustomTransforms) { + throw new ImageTransformConfigError("custom transforms are disabled") + } + + if (!CUSTOM_TRANSFORM_CONFIG.formats.includes(input.format)) { + throw new ImageTransformConfigError(`format ${input.format} is not allowed for custom transforms`) + } + + if (!input.width || input.width > CUSTOM_TRANSFORM_CONFIG.maxWidth) { + throw new ImageTransformConfigError(`custom width must be between 1 and ${CUSTOM_TRANSFORM_CONFIG.maxWidth}`) + } + + const height = input.height ?? 0 + + if (height < 0 || height > CUSTOM_TRANSFORM_CONFIG.maxHeight) { + throw new ImageTransformConfigError(`custom height must be between 0 and ${CUSTOM_TRANSFORM_CONFIG.maxHeight}`) + } + + const quality = input.quality ?? CUSTOM_TRANSFORM_CONFIG.quality + + if (!Number.isSafeInteger(quality) || quality < 1 || quality > 100) { + throw new ImageTransformConfigError("custom quality must be between 1 and 100") + } + + return { + format: input.format, + height, + mode: "custom", + preset: input.preset, + quality, + requestedFormat, + resize: input.resize ?? "fit", + width: input.width, + } +} + +function normalizePresetQuality(value: number | null | undefined, preset: ImagePreset) { + const quality = value ?? preset.quality + + if (!preset.qualities.includes(quality)) { + throw new ImageTransformConfigError(`quality ${quality} is not allowed for preset`) + } + + return quality +} + +function getAllowedFormats(presetName: string, allowCustomTransforms: boolean) { + const preset = getImagePreset(presetName) + + if (preset) { + return preset.formats + } + + if (presetName === CUSTOM_PRESET_NAME && allowCustomTransforms) { + return CUSTOM_TRANSFORM_CONFIG.formats + } + + throw new ImageTransformConfigError(`unknown image preset: ${presetName}`) +} + +function isActualImageFormat(value: string): value is ActualImageFormat { + return value === "avif" || value === "webp" || value === "jpg" || value === "png" +} + +function isRequestedImageFormat(value: string): value is RequestedImageFormat { + return value === "auto" || isActualImageFormat(value) +} diff --git a/packages/image-config/tsconfig.build.json b/packages/image-config/tsconfig.build.json new file mode 100644 index 0000000..69b2412 --- /dev/null +++ b/packages/image-config/tsconfig.build.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false + }, + "exclude": ["dist", "node_modules", "**/*.spec.ts"] +} diff --git a/packages/image-config/tsconfig.json b/packages/image-config/tsconfig.json new file mode 100644 index 0000000..2583c30 --- /dev/null +++ b/packages/image-config/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "allowSyntheticDefaultImports": true, + "declaration": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "lib": ["ES2023"], + "module": "Node16", + "moduleResolution": "Node16", + "noUncheckedIndexedAccess": true, + "outDir": "./dist", + "rootDir": "./src", + "skipLibCheck": true, + "sourceMap": true, + "strict": true, + "target": "ES2023", + "types": ["node"] + }, + "include": ["src/**/*.ts"], + "exclude": ["dist", "node_modules"] +} diff --git a/packages/queue/package.json b/packages/queue/package.json index 8a1eab3..b97f3b7 100644 --- a/packages/queue/package.json +++ b/packages/queue/package.json @@ -2,10 +2,11 @@ "name": "@image-platform/queue", "version": "0.1.0", "private": true, - "type": "module", "exports": { ".": { "types": "./src/index.ts", + "require": "./dist/index.js", + "import": "./dist/index.js", "default": "./dist/index.js" } }, @@ -15,7 +16,11 @@ "typecheck": "tsc --noEmit -p tsconfig.json" }, "devDependencies": { + "@types/amqplib": "^0.10.8", "@types/node": "^25.6.0", "typescript": "^6.0.3" + }, + "dependencies": { + "amqplib": "^1.0.4" } } diff --git a/packages/queue/src/amqp.ts b/packages/queue/src/amqp.ts new file mode 100644 index 0000000..8983960 --- /dev/null +++ b/packages/queue/src/amqp.ts @@ -0,0 +1,29 @@ +import type { Channel } from "amqplib" + +import type { GenerateVariantJob } from "./jobs.js" +import type { QueueTopology } from "./topology.js" + +export async function assertQueueTopology(channel: Channel, topology: QueueTopology) { + await channel.assertExchange(topology.jobsExchange, "direct", { durable: true }) + await channel.assertExchange(topology.jobsDeadLetterExchange, "direct", { durable: true }) + await channel.assertQueue(topology.generateVariantQueue, { + deadLetterExchange: topology.jobsDeadLetterExchange, + deadLetterRoutingKey: topology.generateVariantDeadLetterRoutingKey, + durable: true, + }) + await channel.assertQueue(topology.generateVariantDeadLetterQueue, { durable: true }) + await channel.bindQueue(topology.generateVariantQueue, topology.jobsExchange, topology.generateVariantRoutingKey) + await channel.bindQueue( + topology.generateVariantDeadLetterQueue, + topology.jobsDeadLetterExchange, + topology.generateVariantDeadLetterRoutingKey, + ) +} + +export function publishGenerateVariantJob(channel: Channel, topology: QueueTopology, job: GenerateVariantJob) { + return channel.publish(topology.jobsExchange, topology.generateVariantRoutingKey, Buffer.from(JSON.stringify(job)), { + contentType: "application/json", + deliveryMode: 2, + persistent: true, + }) +} diff --git a/packages/queue/src/index.ts b/packages/queue/src/index.ts index f90efe6..d8c8ca7 100644 --- a/packages/queue/src/index.ts +++ b/packages/queue/src/index.ts @@ -1,2 +1,3 @@ +export * from "./amqp.js" export * from "./jobs.js" export * from "./topology.js" diff --git a/packages/queue/tsconfig.json b/packages/queue/tsconfig.json index 6bc598e..2583c30 100644 --- a/packages/queue/tsconfig.json +++ b/packages/queue/tsconfig.json @@ -5,8 +5,8 @@ "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "lib": ["ES2023"], - "module": "NodeNext", - "moduleResolution": "NodeNext", + "module": "Node16", + "moduleResolution": "Node16", "noUncheckedIndexedAccess": true, "outDir": "./dist", "rootDir": "./src", diff --git a/packages/storage/package.json b/packages/storage/package.json index bc753a9..75bceb5 100644 --- a/packages/storage/package.json +++ b/packages/storage/package.json @@ -2,10 +2,11 @@ "name": "@image-platform/storage", "version": "0.1.0", "private": true, - "type": "module", "exports": { ".": { "types": "./src/index.ts", + "require": "./dist/index.js", + "import": "./dist/index.js", "default": "./dist/index.js" } }, diff --git a/packages/storage/src/config.ts b/packages/storage/src/config.ts index 01d7bfe..a8f76da 100644 --- a/packages/storage/src/config.ts +++ b/packages/storage/src/config.ts @@ -8,11 +8,15 @@ export type StorageConfig = { } export function loadStorageConfigFromEnv(env: NodeJS.ProcessEnv = process.env): StorageConfig { + if (!env.S3_BUCKET) { + throw new Error("S3_BUCKET is required") + } + return { accessKeyId: normalizeOptionalString(env.S3_ACCESS_KEY_ID), - bucket: env.S3_BUCKET ?? "image-platform", + bucket: env.S3_BUCKET, endpoint: normalizeOptionalString(env.S3_ENDPOINT), - forcePathStyle: parseBoolean(env.S3_FORCE_PATH_STYLE, true), + forcePathStyle: parseBoolean(env.S3_FORCE_PATH_STYLE, false), region: env.S3_REGION ?? "us-east-1", secretAccessKey: normalizeOptionalString(env.S3_SECRET_ACCESS_KEY), } diff --git a/packages/storage/src/index.ts b/packages/storage/src/index.ts index f220915..52a10f1 100644 --- a/packages/storage/src/index.ts +++ b/packages/storage/src/index.ts @@ -1,3 +1,4 @@ export * from "./client.js" export * from "./config.js" export * from "./keys.js" +export * from "./objects.js" diff --git a/packages/storage/src/objects.ts b/packages/storage/src/objects.ts new file mode 100644 index 0000000..d8eee47 --- /dev/null +++ b/packages/storage/src/objects.ts @@ -0,0 +1,91 @@ +import { GetObjectCommand, HeadObjectCommand, PutObjectCommand, type S3Client } from "@aws-sdk/client-s3" + +export type StoredObject = { + body: Buffer + contentLength: number | null + contentType: string | null + etag: string | null +} + +export async function getObjectBuffer(client: S3Client, bucket: string, key: string): Promise { + try { + const response = await client.send(new GetObjectCommand({ Bucket: bucket, Key: key })) + + if (!response.Body) { + throw new Error(`S3 object ${key} has no body`) + } + + const body = await streamToBuffer(response.Body as AsyncIterable) + + return { + body, + contentLength: response.ContentLength ?? body.length, + contentType: response.ContentType ?? null, + etag: response.ETag ?? null, + } + } catch (error) { + if (isS3NotFound(error)) { + return null + } + + throw error + } +} + +export async function objectExists(client: S3Client, bucket: string, key: string) { + try { + await client.send(new HeadObjectCommand({ Bucket: bucket, Key: key })) + return true + } catch (error) { + if (isS3NotFound(error)) { + return false + } + + throw error + } +} + +export async function putObjectBuffer(input: { + body: Buffer + bucket: string + cacheControl?: string + client: S3Client + contentType: string + key: string +}) { + return input.client.send( + new PutObjectCommand({ + Body: input.body, + Bucket: input.bucket, + CacheControl: input.cacheControl, + ContentType: input.contentType, + Key: input.key, + }), + ) +} + +function isS3NotFound(error: unknown) { + if (!(error instanceof Error)) { + return false + } + + const withMetadata = error as Error & { $metadata?: { httpStatusCode?: number }; Code?: string; code?: string } + + return ( + withMetadata.$metadata?.httpStatusCode === 404 || + error.name === "NoSuchKey" || + error.name === "NotFound" || + withMetadata.Code === "NoSuchKey" || + withMetadata.code === "NoSuchKey" + ) +} + +async function streamToBuffer(stream: AsyncIterable) { + const chunks: Buffer[] = [] + + for await (const chunk of stream) { + chunks.push(Buffer.from(chunk)) + } + + return Buffer.concat(chunks) +} diff --git a/packages/storage/tsconfig.json b/packages/storage/tsconfig.json index 6bc598e..2583c30 100644 --- a/packages/storage/tsconfig.json +++ b/packages/storage/tsconfig.json @@ -5,8 +5,8 @@ "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "lib": ["ES2023"], - "module": "NodeNext", - "moduleResolution": "NodeNext", + "module": "Node16", + "moduleResolution": "Node16", "noUncheckedIndexedAccess": true, "outDir": "./dist", "rootDir": "./src", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c6b1151..850c8aa 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -38,6 +38,18 @@ importers: apps/backend: dependencies: + '@image-platform/database': + specifier: workspace:* + version: link:../../packages/database + '@image-platform/image-config': + specifier: workspace:* + version: link:../../packages/image-config + '@image-platform/queue': + specifier: workspace:* + version: link:../../packages/queue + '@image-platform/storage': + specifier: workspace:* + version: link:../../packages/storage '@nestjs/common': specifier: ^11.0.0 version: 11.1.19(reflect-metadata@0.2.2)(rxjs@7.8.2) @@ -50,6 +62,12 @@ importers: '@nestjs/swagger': specifier: ^11.0.0 version: 11.4.2(@nestjs/common@11.1.19(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.19)(reflect-metadata@0.2.2) + amqplib: + specifier: ^1.0.4 + version: 1.0.4 + drizzle-orm: + specifier: ^0.45.2 + version: 0.45.2(@types/pg@8.20.0)(pg@8.20.0) reflect-metadata: specifier: ^0.2.2 version: 0.2.2 @@ -66,6 +84,12 @@ importers: '@nestjs/schematics': specifier: ^11.0.0 version: 11.1.0(chokidar@4.0.3)(typescript@5.9.3) + '@types/amqplib': + specifier: ^0.10.8 + version: 0.10.8 + '@types/express': + specifier: ^5.0.6 + version: 5.0.6 '@types/node': specifier: ^24.0.0 version: 24.12.2 @@ -78,6 +102,9 @@ importers: apps/gateway: dependencies: + '@image-platform/image-config': + specifier: workspace:* + version: link:../../packages/image-config fastify: specifier: ^5.8.5 version: 5.8.5 @@ -97,6 +124,9 @@ importers: '@image-platform/database': specifier: workspace:* version: link:../../packages/database + '@image-platform/image-config': + specifier: workspace:* + version: link:../../packages/image-config '@image-platform/queue': specifier: workspace:* version: link:../../packages/queue @@ -106,6 +136,9 @@ importers: amqplib: specifier: ^1.0.4 version: 1.0.4 + drizzle-orm: + specifier: ^0.45.2 + version: 0.45.2(@types/pg@8.20.0)(pg@8.20.0) devDependencies: '@types/amqplib': specifier: ^0.10.8 @@ -142,7 +175,7 @@ importers: specifier: ^6.0.3 version: 6.0.3 - packages/queue: + packages/image-config: devDependencies: '@types/node': specifier: ^25.6.0 @@ -151,6 +184,22 @@ importers: specifier: ^6.0.3 version: 6.0.3 + packages/queue: + dependencies: + amqplib: + specifier: ^1.0.4 + version: 1.0.4 + devDependencies: + '@types/amqplib': + specifier: ^0.10.8 + version: 0.10.8 + '@types/node': + specifier: ^25.6.0 + version: 25.6.0 + typescript: + specifier: ^6.0.3 + version: 6.0.3 + packages/storage: dependencies: '@aws-sdk/client-s3':