feat: добавить генерацию image variants

- добавлен shared config presets, custom transforms и allowlist hosts
- реализованы Backend endpoints для assets, presets и variants
- добавлена orchestration через PostgreSQL, RabbitMQ, S3 и worker
- обновлён Gateway read-through flow с L1 cache и корректным Vary: Accept
- добавлена миграция resize_mode для variants lookup
- обновлены dev scripts, env template, lockfile и документация
This commit is contained in:
2026-05-05 13:25:28 +03:00
parent bcadb85a83
commit 1c0e8277a3
59 changed files with 3526 additions and 143 deletions

View File

@@ -29,6 +29,14 @@ PUBLIC_IMAGE_BASE_URL=http://localhost:8888
# Gateway proxies /api and Swagger routes to this upstream. # Gateway proxies /api and Swagger routes to this upstream.
GATEWAY_BACKEND_UPSTREAM=http://localhost:3001 GATEWAY_BACKEND_UPSTREAM=http://localhost:3001
GATEWAY_L1_MAX_ENTRIES=256
GATEWAY_L1_TTL_MS=600000
# MVP dev mode: mock source host allowlist without DB/admin CRUD.
SOURCE_HOST_ALLOW_ALL=false
SOURCE_ALLOWED_HOSTS=storage.yandexcloud.net
IMAGE_ALLOW_CUSTOM_TRANSFORMS=true
IMAGE_ENSURE_WAIT_MS=15000
# Dev imgproxy is exposed only on localhost. # Dev imgproxy is exposed only on localhost.
IMGPROXY_PORT=18080 IMGPROXY_PORT=18080

View File

@@ -4,7 +4,7 @@ Image Platform - отдельная площадка для управления
## Статус ## Статус
Сейчас создан базовый monorepo, dev-инфраструктура, NestJS backend с Swagger, чистый Vite React TS admin, Fastify gateway skeleton, Drizzle database package, shared queue/storage packages и worker skeleton. Сейчас создан базовый monorepo, dev-инфраструктура, NestJS backend с Swagger, чистый Vite React TS admin, Fastify gateway, Drizzle database package, shared queue/storage packages и worker. Минимальный read-through flow уже реализован для dev.
## Целевая схема ## Целевая схема
@@ -39,7 +39,7 @@ client
- Fastify gateway - Fastify gateway
- worker - worker
Gateway уже добавлен как JS/Fastify skeleton. Сейчас `/images/*` возвращает `501`, пока не подключены DB/S3/imgproxy. Gateway принимает `/images/{assetId}/v{version}/{preset}?w={width}&q={quality}&f=auto`, выбирает фактический формат по `Accept`, вызывает Backend ensure endpoint и кэширует готовые bytes в L1 memory cache.
```bash ```bash
cp .env.example .env cp .env.example .env
@@ -53,6 +53,32 @@ pnpm gateway:dev
pnpm worker:dev pnpm worker:dev
``` ```
`.env` игнорируется git. Runtime-код не содержит dev credentials fallback: для production нужно передать реальные `DATABASE_URL`, `RABBITMQ_URL`, `S3_*` и `IMGPROXY_UPSTREAM` через окружение.
Минимальный smoke flow:
```bash
curl -sS -X POST http://localhost:3001/api/assets \
-H 'content-type: application/json' \
-d '{"sourceUrl":"https://storage.yandexcloud.net/shared1318/img/1.jpg","publicId":"asset_demo"}'
curl -i "http://localhost:8888/images/asset_demo/v1/card?w=640&q=80&f=auto"
curl -i "http://localhost:8888/images/asset_demo/v1/avatar?f=auto"
```
Сейчас доступны static presets из `packages/image-config`: `card`, `hero`, `avatar`. Произвольный single-image режим доступен через `/images/{assetId}/v{version}/custom?...`, если включён `IMAGE_ALLOW_CUSTOM_TRANSFORMS=true`.
Business API без админки:
```bash
curl -sS http://localhost:3001/api/presets
curl -sS http://localhost:3001/api/assets
curl -sS http://localhost:3001/api/assets/asset_demo/variants
curl -sS -X POST http://localhost:3001/api/assets/asset_demo/variants \
-H 'content-type: application/json' \
-d '{"preset":"card","mode":"family"}'
```
Порты по умолчанию: Порты по умолчанию:
| Сервис | URL | | Сервис | URL |

View File

@@ -4,15 +4,21 @@
"private": true, "private": true,
"scripts": { "scripts": {
"build": "nest build", "build": "nest build",
"dev": "nest start --watch", "dev": "node --env-file-if-exists=../../.env ./node_modules/@nestjs/cli/bin/nest.js start --watch",
"start": "node dist/main.js", "start": "node --env-file-if-exists=../../.env dist/main.js",
"typecheck": "tsc --noEmit -p tsconfig.json" "typecheck": "tsc --noEmit -p tsconfig.json"
}, },
"dependencies": { "dependencies": {
"@image-platform/database": "workspace:*",
"@image-platform/image-config": "workspace:*",
"@image-platform/queue": "workspace:*",
"@image-platform/storage": "workspace:*",
"@nestjs/common": "^11.0.0", "@nestjs/common": "^11.0.0",
"@nestjs/core": "^11.0.0", "@nestjs/core": "^11.0.0",
"@nestjs/platform-express": "^11.0.0", "@nestjs/platform-express": "^11.0.0",
"@nestjs/swagger": "^11.0.0", "@nestjs/swagger": "^11.0.0",
"amqplib": "^1.0.4",
"drizzle-orm": "^0.45.2",
"reflect-metadata": "^0.2.2", "reflect-metadata": "^0.2.2",
"rxjs": "^7.8.1", "rxjs": "^7.8.1",
"swagger-ui-express": "^5.0.1" "swagger-ui-express": "^5.0.1"
@@ -20,6 +26,8 @@
"devDependencies": { "devDependencies": {
"@nestjs/cli": "^11.0.0", "@nestjs/cli": "^11.0.0",
"@nestjs/schematics": "^11.0.0", "@nestjs/schematics": "^11.0.0",
"@types/amqplib": "^0.10.8",
"@types/express": "^5.0.6",
"@types/node": "^24.0.0", "@types/node": "^24.0.0",
"@types/swagger-ui-express": "^4.1.8", "@types/swagger-ui-express": "^4.1.8",
"typescript": "^5.9.0" "typescript": "^5.9.0"

View File

@@ -1,9 +1,17 @@
import { Module } from "@nestjs/common" import { Module } from "@nestjs/common"
import { AssetsController } from "./assets/assets.controller"
import { AssetsService } from "./assets/assets.service"
import { HealthController } from "./health/health.controller" import { HealthController } from "./health/health.controller"
import { DatabaseService } from "./infra/database.service"
import { QueueService } from "./infra/queue.service"
import { StorageService } from "./infra/storage.service"
import { InternalImagesController } from "./internal-images/internal-images.controller" import { InternalImagesController } from "./internal-images/internal-images.controller"
import { InternalImagesService } from "./internal-images/internal-images.service"
import { PresetsController } from "./presets/presets.controller"
@Module({ @Module({
controllers: [HealthController, InternalImagesController], controllers: [HealthController, AssetsController, InternalImagesController, PresetsController],
providers: [AssetsService, DatabaseService, InternalImagesService, QueueService, StorageService],
}) })
export class AppModule {} export class AppModule {}

View File

@@ -0,0 +1,93 @@
import { ApiProperty, ApiPropertyOptional } from "@nestjs/swagger"
export class AssetResponseDto {
@ApiProperty({ description: "Внутренний UUID asset.", example: "59fcf4f6-9891-4df4-8bb7-a4dbe570bb66" })
id!: string
@ApiProperty({ description: "Публичный идентификатор asset.", example: "asset_demo" })
publicId!: string
@ApiProperty({ description: "Текущая версия source image.", example: 1 })
currentVersion!: number
@ApiProperty({ description: "Статус asset.", enum: ["active", "disabled", "deleted"], example: "active" })
status!: string
@ApiProperty({ description: "Source URL текущей версии.", example: "https://storage.yandexcloud.net/shared1318/img/1.jpg" })
sourceUrl!: string
@ApiProperty({ description: "Hostname source URL текущей версии.", example: "storage.yandexcloud.net" })
sourceHost!: string
@ApiProperty({ description: "Дата создания asset.", example: "2026-05-05T12:00:00.000Z" })
createdAt!: string
@ApiProperty({ description: "Дата обновления asset.", example: "2026-05-05T12:00:00.000Z" })
updatedAt!: string
}
export class AssetVariantResponseDto {
@ApiProperty({ description: "Внутренний UUID variant.", example: "7748d24e-5f30-4064-8ee8-4745a4d2aef1" })
id!: string
@ApiProperty({ description: "Preset или `custom`.", example: "card" })
preset!: string
@ApiProperty({ description: "Версия source image, для которой создан variant.", example: 1 })
version!: number
@ApiProperty({ description: "Ширина variant.", example: 640 })
width!: number
@ApiProperty({ description: "Высота variant. `0` означает auto height.", example: 0 })
height!: number
@ApiProperty({ description: "Режим resize.", enum: ["fit", "fill"], example: "fit" })
resize!: string
@ApiProperty({ description: "Качество variant.", example: 80 })
quality!: number
@ApiProperty({ description: "Запрошенный формат.", enum: ["auto", "avif", "webp", "jpg", "png"], example: "webp" })
requestedFormat!: string
@ApiProperty({ description: "Фактический формат bytes.", enum: ["avif", "webp", "jpg", "png"], example: "webp" })
format!: string
@ApiProperty({ description: "Статус генерации.", enum: ["pending", "processing", "ready", "failed"], example: "ready" })
status!: string
@ApiProperty({ description: "Публичный Gateway URL для variant.", example: "http://localhost:8888/images/asset_demo/v1/card?w=640&q=80&f=webp" })
url!: string
@ApiProperty({ description: "S3 key variant object.", example: "variants/asset_demo/v1/abc.webp" })
s3Key!: string
@ApiPropertyOptional({ description: "Content-Type готового object.", example: "image/webp" })
contentType!: string | null
@ApiPropertyOptional({ description: "Размер готового object в bytes.", example: 71844 })
sizeBytes!: number | null
@ApiPropertyOptional({ description: "Ошибка последней генерации, если status=`failed`." })
error!: string | null
@ApiProperty({ description: "Дата создания variant.", example: "2026-05-05T12:00:00.000Z" })
createdAt!: string
@ApiProperty({ description: "Дата обновления variant.", example: "2026-05-05T12:00:00.000Z" })
updatedAt!: string
}
export class AssetVariantsResponseDto {
@ApiProperty({ description: "Публичный идентификатор asset.", example: "asset_demo" })
publicId!: string
@ApiProperty({ description: "Список variants.", type: [AssetVariantResponseDto] })
variants!: AssetVariantResponseDto[]
}
export class AssetsListResponseDto {
@ApiProperty({ description: "Список assets.", type: [AssetResponseDto] })
assets!: AssetResponseDto[]
}

View File

@@ -0,0 +1,93 @@
import { Body, Controller, Get, Param, Post, Query } from "@nestjs/common"
import {
ApiBadRequestResponse,
ApiConflictResponse,
ApiCreatedResponse,
ApiNotFoundResponse,
ApiOkResponse,
ApiOperation,
ApiParam,
ApiQuery,
ApiTags,
} from "@nestjs/swagger"
import { AssetsService } from "./assets.service"
import { AssetResponseDto, AssetVariantsResponseDto, AssetsListResponseDto } from "./asset-response.dto"
import { CreateAssetVariantsRequestDto, CreateAssetVariantsResponseDto } from "./create-asset-variants.dto"
import { CreateAssetRequestDto, CreateAssetResponseDto } from "./create-asset.dto"
@ApiTags("assets")
@Controller("assets")
export class AssetsController {
constructor(private readonly assets: AssetsService) {}
@Get()
@ApiOperation({
summary: "получить список assets",
description: "Возвращает последние зарегистрированные assets вместе с source URL текущей версии.",
})
@ApiQuery({ description: "Максимальное количество assets в ответе.", example: 50, name: "limit", required: false })
@ApiQuery({ description: "Смещение для простого paging.", example: 0, name: "offset", required: false })
@ApiOkResponse({ description: "Список assets возвращён.", type: AssetsListResponseDto })
listAssets(@Query("limit") limit?: string, @Query("offset") offset?: string): Promise<AssetsListResponseDto> {
return this.assets.listAssets({ limit, offset })
}
@Post()
@ApiOperation({
summary: "зарегистрировать исходное изображение",
description:
"Создаёт asset и первую версию source image. Source URL сохраняется в PostgreSQL, а публичный image URL строится через Gateway без раскрытия исходной ссылки клиенту.",
})
@ApiCreatedResponse({ description: "Asset создан, версия source image зарегистрирована.", type: CreateAssetResponseDto })
@ApiBadRequestResponse({ description: "Некорректный sourceUrl, publicId или source host запрещён настройками." })
@ApiConflictResponse({ description: "Asset с таким publicId уже существует." })
createAsset(@Body() request: CreateAssetRequestDto): Promise<CreateAssetResponseDto> {
return this.assets.createAsset(request)
}
@Get(":publicId")
@ApiOperation({
summary: "получить asset по publicId",
description: "Возвращает metadata asset и source URL текущей версии.",
})
@ApiParam({ description: "Публичный идентификатор asset.", example: "asset_demo", name: "publicId" })
@ApiOkResponse({ description: "Asset найден.", type: AssetResponseDto })
@ApiNotFoundResponse({ description: "Asset не найден." })
getAsset(@Param("publicId") publicId: string): Promise<AssetResponseDto> {
return this.assets.getAsset(publicId)
}
@Get(":publicId/variants")
@ApiOperation({
summary: "получить variants asset",
description: "Возвращает variants asset: preset/custom параметры, status, S3 key, public URL и ошибку генерации, если она была.",
})
@ApiParam({ description: "Публичный идентификатор asset.", example: "asset_demo", name: "publicId" })
@ApiQuery({ description: "Версия source image. Если не передана, возвращаются variants всех версий.", example: 1, name: "version", required: false })
@ApiOkResponse({ description: "Variants возвращены.", type: AssetVariantsResponseDto })
@ApiNotFoundResponse({ description: "Asset не найден." })
listAssetVariants(
@Param("publicId") publicId: string,
@Query("version") version?: string,
): Promise<AssetVariantsResponseDto> {
return this.assets.listAssetVariants(publicId, version)
}
@Post(":publicId/variants")
@ApiOperation({
summary: "поставить generation jobs для variants",
description:
"Business endpoint для явной подготовки variants. В режиме `single` создаёт один variant, в режиме `family` создаёт набор variants preset по всем разрешённым widths/formats. Endpoint не ждёт bytes, а возвращает созданные/переиспользованные rows и public URLs.",
})
@ApiParam({ description: "Публичный идентификатор asset.", example: "asset_demo", name: "publicId" })
@ApiCreatedResponse({ description: "Variants созданы или переиспользованы, jobs поставлены при необходимости.", type: CreateAssetVariantsResponseDto })
@ApiBadRequestResponse({ description: "Некорректный preset/custom transform config." })
@ApiNotFoundResponse({ description: "Asset или version не найдены." })
createAssetVariants(
@Param("publicId") publicId: string,
@Body() request: CreateAssetVariantsRequestDto,
): Promise<CreateAssetVariantsResponseDto> {
return this.assets.createAssetVariants(publicId, request)
}
}

View File

@@ -0,0 +1,611 @@
import { BadRequestException, ConflictException, Injectable, NotFoundException } from "@nestjs/common"
import { imageAssets, imageAssetVersions, imageVariants } from "@image-platform/database"
import {
CUSTOM_PRESET_NAME,
ImageTransformConfigError,
getImagePreset,
isAllowedSourceHost,
loadAllowedSourceHostsFromEnv,
normalizeImageTransform,
parseBooleanFlag,
type ActualImageFormat,
type NormalizedImageTransform,
} from "@image-platform/image-config"
import { buildVariantImageKey } from "@image-platform/storage"
import { and, desc, eq } from "drizzle-orm"
import { createHash, randomUUID } from "node:crypto"
import { DatabaseService } from "../infra/database.service"
import { QueueService } from "../infra/queue.service"
import type { AssetResponseDto, AssetVariantResponseDto, AssetVariantsResponseDto, AssetsListResponseDto } from "./asset-response.dto"
import type { CreateAssetVariantsRequestDto, CreateAssetVariantsResponseDto } from "./create-asset-variants.dto"
import type { CreateAssetRequestDto, CreateAssetResponseDto } from "./create-asset.dto"
import { normalizeSourceUrl } from "./source-url"
type AssetVersionRow = {
assetId: string
currentVersion: number
publicId: string
status: "active" | "deleted" | "disabled"
versionId: string
version: number
}
type VariantRow = typeof imageVariants.$inferSelect
type VariantTransform = NormalizedImageTransform & {
version: number
}
@Injectable()
export class AssetsService {
private readonly allowedHosts = loadAllowedSourceHostsFromEnv()
private readonly allowCustomTransforms = parseBooleanFlag(process.env.IMAGE_ALLOW_CUSTOM_TRANSFORMS, false)
private readonly allowUnregisteredHosts = parseBooleanFlag(process.env.SOURCE_HOST_ALLOW_ALL, false)
private readonly publicImageBaseUrl = process.env.PUBLIC_IMAGE_BASE_URL ?? "http://localhost:8888"
constructor(
private readonly database: DatabaseService,
private readonly queue: QueueService,
) {}
async listAssets(input: { limit?: string; offset?: string }): Promise<AssetsListResponseDto> {
const limit = parsePaginationInteger(input.limit, 50, 1, 100)
const offset = parsePaginationInteger(input.offset, 0, 0, 10_000)
const rows = await this.database.db
.select({
createdAt: imageAssets.createdAt,
currentVersion: imageAssets.currentVersion,
id: imageAssets.id,
publicId: imageAssets.publicId,
sourceHost: imageAssetVersions.sourceHost,
sourceUrl: imageAssetVersions.sourceUrl,
status: imageAssets.status,
updatedAt: imageAssets.updatedAt,
})
.from(imageAssets)
.innerJoin(
imageAssetVersions,
and(eq(imageAssetVersions.assetId, imageAssets.id), eq(imageAssetVersions.version, imageAssets.currentVersion)),
)
.orderBy(desc(imageAssets.createdAt))
.limit(limit)
.offset(offset)
return {
assets: rows.map(mapAssetResponse),
}
}
async createAsset(request: CreateAssetRequestDto): Promise<CreateAssetResponseDto> {
const source = normalizeSourceUrl(request.sourceUrl)
await this.assertAllowedHost(source.hostname)
const publicId = request.publicId ? normalizePublicId(request.publicId) : generatePublicId()
const sourceHash = createHash("sha256").update(source.sourceUrl).digest("hex")
try {
const result = await this.database.db.transaction(async (tx) => {
const [asset] = await tx
.insert(imageAssets)
.values({ publicId })
.returning({ id: imageAssets.id, publicId: imageAssets.publicId })
if (!asset) {
throw new Error("failed to create image asset")
}
const [version] = await tx
.insert(imageAssetVersions)
.values({
assetId: asset.id,
sourceHash,
sourceHost: source.hostname,
sourceUrl: source.sourceUrl,
version: 1,
})
.returning({ sourceHost: imageAssetVersions.sourceHost, version: imageAssetVersions.version })
if (!version) {
throw new Error("failed to create image asset version")
}
return { asset, version }
})
return {
id: result.asset.id,
imageBasePath: `/images/${result.asset.publicId}/v${result.version.version}/card`,
publicId: result.asset.publicId,
sourceHost: result.version.sourceHost,
version: result.version.version,
}
} catch (error) {
if (isUniqueViolation(error)) {
throw new ConflictException("publicId already exists")
}
throw error
}
}
async getAsset(publicId: string): Promise<AssetResponseDto> {
const asset = await this.loadAsset(publicId)
return mapAssetResponse(asset)
}
async listAssetVariants(publicId: string, versionInput?: string): Promise<AssetVariantsResponseDto> {
const asset = await this.loadAssetVersion(publicId, parseOptionalVersion(versionInput))
const conditions = [eq(imageVariants.assetId, asset.assetId)]
if (versionInput !== undefined) {
conditions.push(eq(imageVariants.assetVersion, asset.version))
}
const variants = await this.database.db
.select()
.from(imageVariants)
.where(and(...conditions))
.orderBy(desc(imageVariants.createdAt))
return {
publicId: asset.publicId,
variants: variants.map((variant) => this.mapVariantResponse(asset.publicId, variant)),
}
}
async createAssetVariants(
publicId: string,
request: CreateAssetVariantsRequestDto,
): Promise<CreateAssetVariantsResponseDto> {
const asset = await this.loadAssetVersion(publicId, request.version)
const transforms = this.buildVariantTransforms(request, asset.version)
const variants: AssetVariantResponseDto[] = []
for (const transform of transforms) {
const variant = await this.findOrCreateVariant(asset, transform)
if (variant.status === "failed") {
const pending = await this.markVariantPending(variant.id)
this.queue.publishGenerateVariant(pending.id)
variants.push(this.mapVariantResponse(asset.publicId, pending))
continue
}
if (variant.status === "pending") {
this.queue.publishGenerateVariant(variant.id)
}
variants.push(this.mapVariantResponse(asset.publicId, variant))
}
return {
publicId: asset.publicId,
variants,
version: asset.version,
}
}
private async loadAsset(publicId: string) {
const normalizedPublicId = normalizePublicId(publicId)
const [asset] = await this.database.db
.select({
createdAt: imageAssets.createdAt,
currentVersion: imageAssets.currentVersion,
id: imageAssets.id,
publicId: imageAssets.publicId,
sourceHost: imageAssetVersions.sourceHost,
sourceUrl: imageAssetVersions.sourceUrl,
status: imageAssets.status,
updatedAt: imageAssets.updatedAt,
})
.from(imageAssets)
.innerJoin(
imageAssetVersions,
and(eq(imageAssetVersions.assetId, imageAssets.id), eq(imageAssetVersions.version, imageAssets.currentVersion)),
)
.where(eq(imageAssets.publicId, normalizedPublicId))
.limit(1)
if (!asset) {
throw new NotFoundException("asset not found")
}
return asset
}
private async loadAssetVersion(publicId: string, versionInput: number | null | undefined): Promise<AssetVersionRow> {
const normalizedPublicId = normalizePublicId(publicId)
const [asset] = await this.database.db
.select({
assetId: imageAssets.id,
currentVersion: imageAssets.currentVersion,
publicId: imageAssets.publicId,
status: imageAssets.status,
})
.from(imageAssets)
.where(eq(imageAssets.publicId, normalizedPublicId))
.limit(1)
if (!asset || asset.status !== "active") {
throw new NotFoundException("asset not found")
}
const version = versionInput ?? asset.currentVersion
const [assetVersion] = await this.database.db
.select({ id: imageAssetVersions.id, version: imageAssetVersions.version })
.from(imageAssetVersions)
.where(and(eq(imageAssetVersions.assetId, asset.assetId), eq(imageAssetVersions.version, version)))
.limit(1)
if (!assetVersion) {
throw new NotFoundException("asset version not found")
}
return {
...asset,
version: assetVersion.version,
versionId: assetVersion.id,
}
}
private buildVariantTransforms(request: CreateAssetVariantsRequestDto, version: number): VariantTransform[] {
const mode = request.mode ?? "single"
if (mode !== "single" && mode !== "family") {
throw new BadRequestException("mode must be single or family")
}
if (mode === "family") {
return this.buildFamilyTransforms(request, version)
}
const format = request.format ?? selectDefaultFormat(request.preset, this.allowCustomTransforms)
return [
this.normalizeTransform({
format,
height: request.height,
preset: request.preset,
quality: request.quality,
resize: request.resize,
version,
width: request.width,
}),
]
}
private buildFamilyTransforms(request: CreateAssetVariantsRequestDto, version: number): VariantTransform[] {
if (request.preset === CUSTOM_PRESET_NAME) {
throw new BadRequestException("custom transforms do not support family mode")
}
const preset = getImagePreset(request.preset)
if (!preset) {
throw new BadRequestException(`unknown image preset: ${request.preset}`)
}
if (request.width !== undefined || request.height !== undefined || request.resize !== undefined) {
throw new BadRequestException("width, height and resize are not accepted in family mode")
}
const formats = [...new Set(request.formats?.length ? request.formats : preset.formats)]
for (const format of formats) {
if (!preset.formats.includes(format)) {
throw new BadRequestException(`format ${format} is not allowed for preset ${request.preset}`)
}
}
const widths = preset.mode === "fixed" ? [preset.width] : preset.widths
if (!widths?.length) {
throw new BadRequestException(`preset ${request.preset} has no widths configured`)
}
return widths.flatMap((width) =>
formats.map((format) =>
this.normalizeTransform({
format,
preset: request.preset,
quality: request.quality,
version,
width: preset.mode === "fixed" ? undefined : width,
}),
),
)
}
private normalizeTransform(input: {
format: ActualImageFormat
height?: number
preset: string
quality?: number
resize?: "fill" | "fit"
version: number
width?: number
}): VariantTransform {
try {
const transform = normalizeImageTransform({
allowCustomTransforms: this.allowCustomTransforms,
format: input.format,
height: input.height,
preset: input.preset,
quality: input.quality,
requestedFormat: input.format,
resize: input.resize,
width: input.width,
})
return {
...transform,
version: input.version,
}
} catch (error) {
if (error instanceof ImageTransformConfigError) {
throw new BadRequestException(error.message)
}
throw error
}
}
private async findOrCreateVariant(asset: AssetVersionRow, transform: VariantTransform): Promise<VariantRow> {
const existing = await this.findVariant(asset.assetId, transform)
if (existing) {
return existing
}
const variantHash = createVariantHash(asset.publicId, transform)
const s3Key = buildVariantImageKey({
assetId: asset.publicId,
format: transform.format,
variantHash,
version: asset.version,
})
const [created] = await this.database.db
.insert(imageVariants)
.values({
assetId: asset.assetId,
assetVersion: asset.version,
assetVersionId: asset.versionId,
format: transform.format,
height: transform.height,
preset: transform.preset,
quality: transform.quality,
requestedFormat: transform.requestedFormat,
resizeMode: transform.resize,
s3Key,
status: "pending",
variantHash,
width: transform.width,
})
.onConflictDoNothing({
target: [
imageVariants.assetId,
imageVariants.assetVersion,
imageVariants.preset,
imageVariants.width,
imageVariants.height,
imageVariants.resizeMode,
imageVariants.quality,
imageVariants.format,
],
})
.returning()
if (created) {
return created
}
const raced = await this.findVariant(asset.assetId, transform)
if (!raced) {
throw new Error("failed to create image variant")
}
return raced
}
private async findVariant(assetId: string, transform: VariantTransform): Promise<VariantRow | null> {
const [variant] = await this.database.db
.select()
.from(imageVariants)
.where(
and(
eq(imageVariants.assetId, assetId),
eq(imageVariants.assetVersion, transform.version),
eq(imageVariants.preset, transform.preset),
eq(imageVariants.width, transform.width),
eq(imageVariants.height, transform.height),
eq(imageVariants.resizeMode, transform.resize),
eq(imageVariants.quality, transform.quality),
eq(imageVariants.format, transform.format),
),
)
.limit(1)
return variant ?? null
}
private async markVariantPending(variantId: string): Promise<VariantRow> {
const [variant] = await this.database.db
.update(imageVariants)
.set({ error: null, status: "pending", updatedAt: new Date() })
.where(eq(imageVariants.id, variantId))
.returning()
if (!variant) {
throw new NotFoundException("variant not found")
}
return variant
}
private mapVariantResponse(publicId: string, variant: VariantRow): AssetVariantResponseDto {
return {
contentType: variant.contentType,
createdAt: variant.createdAt.toISOString(),
error: variant.error,
format: variant.format,
height: variant.height ?? 0,
id: variant.id,
preset: variant.preset,
quality: variant.quality,
requestedFormat: variant.requestedFormat,
resize: variant.resizeMode,
s3Key: variant.s3Key,
sizeBytes: variant.sizeBytes,
status: variant.status,
updatedAt: variant.updatedAt.toISOString(),
url: buildPublicImageUrl(this.publicImageBaseUrl, publicId, variant),
version: variant.assetVersion,
width: variant.width,
}
}
private async assertAllowedHost(hostname: string) {
if (this.allowUnregisteredHosts) {
return
}
if (!isAllowedSourceHost(hostname, this.allowedHosts)) {
throw new BadRequestException("sourceUrl host is not allowed")
}
}
}
function mapAssetResponse(row: {
createdAt: Date
currentVersion: number
id: string
publicId: string
sourceHost: string
sourceUrl: string
status: "active" | "deleted" | "disabled"
updatedAt: Date
}): AssetResponseDto {
return {
createdAt: row.createdAt.toISOString(),
currentVersion: row.currentVersion,
id: row.id,
publicId: row.publicId,
sourceHost: row.sourceHost,
sourceUrl: row.sourceUrl,
status: row.status,
updatedAt: row.updatedAt.toISOString(),
}
}
function generatePublicId() {
return `asset_${randomUUID().replaceAll("-", "").slice(0, 16)}`
}
function normalizePublicId(publicId: string) {
const normalized = publicId.trim()
if (!/^[a-zA-Z0-9_-]{3,128}$/.test(normalized)) {
throw new BadRequestException("publicId must be 3-128 chars and contain only letters, digits, _ or -")
}
return normalized
}
function isUniqueViolation(error: unknown) {
return typeof error === "object" && error !== null && "code" in error && error.code === "23505"
}
function parsePaginationInteger(value: string | undefined, fallback: number, min: number, max: number) {
if (value === undefined) {
return fallback
}
if (!/^\d+$/.test(value)) {
throw new BadRequestException("pagination params must be integers")
}
const parsed = Number.parseInt(value, 10)
if (!Number.isSafeInteger(parsed) || parsed < min || parsed > max) {
throw new BadRequestException(`pagination param must be between ${min} and ${max}`)
}
return parsed
}
function parseOptionalVersion(value: string | undefined) {
if (value === undefined) {
return undefined
}
if (!/^\d+$/.test(value)) {
throw new BadRequestException("version must be a positive integer")
}
const parsed = Number.parseInt(value, 10)
if (!Number.isSafeInteger(parsed) || parsed < 1) {
throw new BadRequestException("version must be a positive integer")
}
return parsed
}
function selectDefaultFormat(preset: string, allowCustomTransforms: boolean): ActualImageFormat {
const config = getImagePreset(preset)
if (config) {
return config.formats.includes("webp") ? "webp" : config.formats[0]
}
if (preset === CUSTOM_PRESET_NAME && allowCustomTransforms) {
return "webp"
}
throw new BadRequestException(`unknown image preset: ${preset}`)
}
function createVariantHash(publicId: string, transform: VariantTransform) {
return createHash("sha256")
.update(
[
publicId,
transform.version,
transform.preset,
transform.width,
transform.height,
transform.resize,
transform.quality,
transform.format,
].join(":"),
)
.digest("hex")
.slice(0, 32)
}
function buildPublicImageUrl(baseUrl: string, publicId: string, variant: VariantRow) {
const url = new URL(`/images/${publicId}/v${variant.assetVersion}/${variant.preset}`, baseUrl)
const isFixedPresetUrl = variant.preset !== CUSTOM_PRESET_NAME && variant.height && variant.height > 0
if (!isFixedPresetUrl || variant.preset === CUSTOM_PRESET_NAME) {
url.searchParams.set("w", variant.width.toString())
}
if (variant.preset === CUSTOM_PRESET_NAME && variant.height) {
url.searchParams.set("h", variant.height.toString())
url.searchParams.set("fit", variant.resizeMode)
}
if (!isFixedPresetUrl || variant.quality !== 80) {
url.searchParams.set("q", variant.quality.toString())
}
url.searchParams.set("f", variant.format)
return url.toString()
}

View File

@@ -0,0 +1,43 @@
import { ApiProperty, ApiPropertyOptional } from "@nestjs/swagger"
import { AssetVariantResponseDto } from "./asset-response.dto"
export class CreateAssetVariantsRequestDto {
@ApiProperty({ description: "Preset для генерации или `custom`.", example: "card" })
preset!: string
@ApiPropertyOptional({ description: "Режим генерации: один variant или вся family preset.", enum: ["single", "family"], example: "single" })
mode?: "family" | "single"
@ApiPropertyOptional({ description: "Версия source image. Если не передана, используется currentVersion asset.", example: 1 })
version?: number
@ApiPropertyOptional({ description: "Ширина variant. Обязательна для responsive preset в mode=`single` и custom.", example: 640 })
width?: number
@ApiPropertyOptional({ description: "Высота variant для custom. `0` или отсутствие означает auto height.", example: 333 })
height?: number
@ApiPropertyOptional({ description: "Качество. Если не передано, берётся из preset/custom config.", example: 80 })
quality?: number
@ApiPropertyOptional({ description: "Фактический формат для single generation.", enum: ["avif", "webp", "jpg", "png"], example: "webp" })
format?: "avif" | "jpg" | "png" | "webp"
@ApiPropertyOptional({ description: "Форматы для family generation. Если не переданы, используются все форматы preset.", enum: ["avif", "webp", "jpg", "png"], isArray: true })
formats?: Array<"avif" | "jpg" | "png" | "webp">
@ApiPropertyOptional({ description: "Resize mode для custom transforms.", enum: ["fit", "fill"], example: "fill" })
resize?: "fill" | "fit"
}
export class CreateAssetVariantsResponseDto {
@ApiProperty({ description: "Публичный идентификатор asset.", example: "asset_demo" })
publicId!: string
@ApiProperty({ description: "Версия source image, для которой поставлены jobs.", example: 1 })
version!: number
@ApiProperty({ description: "Созданные или переиспользованные variants.", type: [AssetVariantResponseDto] })
variants!: AssetVariantResponseDto[]
}

View File

@@ -0,0 +1,36 @@
import { ApiProperty, ApiPropertyOptional } from "@nestjs/swagger"
export class CreateAssetRequestDto {
@ApiPropertyOptional({
description:
"Публичный стабильный идентификатор asset. Если не передан, Backend сгенерирует идентификатор автоматически.",
example: "asset_123",
})
publicId?: string
@ApiProperty({
description: "Постоянная ссылка на исходное изображение. Сейчас поддерживаются только публичные http/https URL.",
example: "https://storage.yandexcloud.net/shared1318/img/1.jpg",
})
sourceUrl!: string
}
export class CreateAssetResponseDto {
@ApiProperty({ description: "Внутренний UUID asset в PostgreSQL.", example: "59fcf4f6-9891-4df4-8bb7-a4dbe570bb66" })
id!: string
@ApiProperty({ description: "Публичный идентификатор asset для Gateway URL.", example: "asset_123" })
publicId!: string
@ApiProperty({ description: "Номер версии source image. Используется в URL как `v{version}`.", example: 1 })
version!: number
@ApiProperty({ description: "Нормализованный hostname исходного изображения.", example: "storage.yandexcloud.net" })
sourceHost!: string
@ApiProperty({
description: "Базовый путь Gateway для запроса variant. Width, quality и format передаются query params.",
example: "/images/asset_123/v1/card",
})
imageBasePath!: string
}

View File

@@ -0,0 +1,65 @@
import { BadRequestException } from "@nestjs/common"
import { isIP } from "node:net"
const LOCAL_HOSTNAMES = new Set(["localhost", "localhost.localdomain"])
export type NormalizedSourceUrl = {
hostname: string
sourceUrl: string
}
export function normalizeSourceUrl(input: string): NormalizedSourceUrl {
let url: URL
try {
url = new URL(input)
} catch {
throw new BadRequestException("sourceUrl must be a valid URL")
}
if (url.protocol !== "http:" && url.protocol !== "https:") {
throw new BadRequestException("sourceUrl must use http or https")
}
const hostname = url.hostname.toLowerCase().replace(/\.$/, "")
if (isPrivateOrLocalHostname(hostname)) {
throw new BadRequestException("sourceUrl host must be public")
}
url.hostname = hostname
url.hash = ""
return {
hostname,
sourceUrl: url.toString(),
}
}
function isPrivateOrLocalHostname(hostname: string) {
if (LOCAL_HOSTNAMES.has(hostname) || hostname.endsWith(".localhost")) {
return true
}
const ipVersion = isIP(hostname)
if (ipVersion === 0) {
return false
}
if (ipVersion === 6) {
return true
}
const [a = 0, b = 0] = hostname.split(".").map((part) => Number.parseInt(part, 10))
return (
a === 0 ||
a === 10 ||
a === 127 ||
(a === 100 && b >= 64 && b <= 127) ||
(a === 169 && b === 254) ||
(a === 172 && b >= 16 && b <= 31) ||
(a === 192 && b === 168)
)
}

View File

@@ -1,9 +1,9 @@
import { ApiProperty } from "@nestjs/swagger" import { ApiProperty } from "@nestjs/swagger"
export class HealthResponseDto { export class HealthResponseDto {
@ApiProperty({ example: "image-platform-api" }) @ApiProperty({ description: "Имя сервиса, который вернул health-check ответ.", example: "image-platform-api" })
service!: string service!: string
@ApiProperty({ example: "ok" }) @ApiProperty({ description: "Текущее состояние сервиса.", example: "ok" })
status!: string status!: string
} }

View File

@@ -7,8 +7,11 @@ import { HealthResponseDto } from "./health-response.dto"
@Controller("health") @Controller("health")
export class HealthController { export class HealthController {
@Get() @Get()
@ApiOperation({ summary: "Проверить состояние API" }) @ApiOperation({
@ApiOkResponse({ type: HealthResponseDto }) summary: "проверить состояние Backend API",
description: "Возвращает простой health-check ответ. Используется для локальной проверки, мониторинга и smoke tests.",
})
@ApiOkResponse({ description: "Backend API доступен и отвечает.", type: HealthResponseDto })
getHealth(): HealthResponseDto { getHealth(): HealthResponseDto {
return { return {
service: "image-platform-api", service: "image-platform-api",

View File

@@ -0,0 +1,14 @@
import { Injectable, OnModuleDestroy } from "@nestjs/common"
import { createDatabase, createDatabasePool } from "@image-platform/database"
import type { Database } from "@image-platform/database"
@Injectable()
export class DatabaseService implements OnModuleDestroy {
private readonly pool = createDatabasePool()
readonly db: Database = createDatabase(this.pool)
async onModuleDestroy() {
await this.pool.end()
}
}

View File

@@ -0,0 +1,52 @@
import { Injectable, OnModuleDestroy, OnModuleInit, ServiceUnavailableException } from "@nestjs/common"
import amqp, { type Channel, type ChannelModel } from "amqplib"
import { assertQueueTopology, loadQueueTopologyFromEnv, publishGenerateVariantJob } from "@image-platform/queue"
import { randomUUID } from "node:crypto"
@Injectable()
export class QueueService implements OnModuleDestroy, OnModuleInit {
private readonly rabbitmqUrl = getRequiredEnv("RABBITMQ_URL")
private readonly topology = loadQueueTopologyFromEnv()
private channel: Channel | null = null
private connection: ChannelModel | null = null
async onModuleInit() {
const connection = await amqp.connect(this.rabbitmqUrl)
const channel = await connection.createChannel()
await assertQueueTopology(channel, this.topology)
this.connection = connection
this.channel = channel
}
async onModuleDestroy() {
await this.channel?.close().catch((error: unknown) => console.error("failed to close RabbitMQ channel", error))
await this.connection?.close().catch((error: unknown) => console.error("failed to close RabbitMQ connection", error))
}
publishGenerateVariant(variantId: string) {
if (!this.channel) {
throw new ServiceUnavailableException("RabbitMQ channel is not ready")
}
const published = publishGenerateVariantJob(this.channel, this.topology, {
jobId: randomUUID(),
variantId,
})
if (!published) {
throw new ServiceUnavailableException("RabbitMQ publish buffer is full")
}
}
}
function getRequiredEnv(name: string) {
const value = process.env[name]
if (!value) {
throw new Error(`${name} is required`)
}
return value
}

View File

@@ -0,0 +1,14 @@
import { Injectable } from "@nestjs/common"
import { createS3Client, getObjectBuffer, loadStorageConfigFromEnv, type StoredObject } from "@image-platform/storage"
@Injectable()
export class StorageService {
private readonly config = loadStorageConfigFromEnv()
private readonly client = createS3Client(this.config)
readonly bucket = this.config.bucket
async getObject(key: string): Promise<StoredObject | null> {
return getObjectBuffer(this.client, this.bucket, key)
}
}

View File

@@ -1,21 +1,43 @@
import { ApiProperty } from "@nestjs/swagger" import { ApiProperty, ApiPropertyOptional } from "@nestjs/swagger"
import type { ActualImageFormat, RequestedImageFormat, ResizeMode } from "@image-platform/image-config"
export class EnsureImageVariantRequestDto { export class EnsureImageVariantRequestDto {
@ApiProperty({ example: "asset_123" }) @ApiProperty({ description: "Публичный идентификатор asset из Gateway URL.", example: "asset_123" })
assetId!: string assetId!: string
@ApiProperty({ example: 4, minimum: 1 }) @ApiProperty({ description: "Версия source image из Gateway URL `v{version}`.", example: 4, minimum: 1 })
version!: number version!: number
@ApiProperty({ example: "card" }) @ApiProperty({ description: "Имя preset трансформации. Сейчас используется как часть variant key.", example: "card" })
preset!: string preset!: string
@ApiProperty({ example: 640, minimum: 1 }) @ApiPropertyOptional({ description: "Целевая ширина variant в пикселях. Обязательна для responsive presets и custom.", example: 640, minimum: 1 })
width!: number width?: number
@ApiProperty({ example: 80, minimum: 1 }) @ApiPropertyOptional({ description: "Целевая высота variant в пикселях. `0` или отсутствие означает auto height.", example: 420, minimum: 0 })
quality!: number height?: number
@ApiProperty({ enum: ["auto", "avif", "webp", "jpg", "png"], example: "auto" }) @ApiPropertyOptional({ description: "Качество сжатия для imgproxy. Если не передано, берётся из preset.", example: 80, minimum: 1 })
format!: "auto" | "avif" | "jpg" | "png" | "webp" quality?: number
@ApiPropertyOptional({
description: "Формат, который запросил клиент. Для `auto` Gateway выбирает фактический формат по `Accept` header.",
enum: ["auto", "avif", "webp", "jpg", "png"],
example: "auto",
})
requestedFormat?: RequestedImageFormat
@ApiPropertyOptional({
description: "Режим resize для custom transforms. Для обычных presets берётся из preset config.",
enum: ["fit", "fill"],
example: "fit",
})
resize?: ResizeMode
@ApiProperty({
description: "Фактический output format после negotiation. Именно этот формат попадает в S3 key и L1 cache key.",
enum: ["avif", "webp", "jpg", "png"],
example: "webp",
})
format!: ActualImageFormat
} }

View File

@@ -1,19 +1,63 @@
import { Body, Controller, NotImplementedException, Post } from "@nestjs/common" import { Body, Controller, Header, Post, Res, StreamableFile } from "@nestjs/common"
import { ApiOperation, ApiResponse, ApiTags } from "@nestjs/swagger" import {
ApiBadGatewayResponse,
ApiBadRequestResponse,
ApiNotFoundResponse,
ApiOkResponse,
ApiOperation,
ApiResponse,
ApiTags,
} from "@nestjs/swagger"
import type { Response } from "express"
import { EnsureImageVariantRequestDto } from "./ensure-image-variant.dto" import { EnsureImageVariantRequestDto } from "./ensure-image-variant.dto"
import { InternalImagesService } from "./internal-images.service"
@ApiTags("internal-images") @ApiTags("internal-images")
@Controller("internal/images") @Controller("internal/images")
export class InternalImagesController { export class InternalImagesController {
constructor(private readonly internalImages: InternalImagesService) {}
@Post("ensure") @Post("ensure")
@ApiOperation({ summary: "Ensure image variant for Gateway L1 miss" }) @ApiOperation({
@ApiResponse({ status: 501, description: "Read-through image pipeline is not implemented yet" }) summary: "подготовить variant изображения для Gateway",
ensureImageVariant(@Body() request: EnsureImageVariantRequestDto): never { description:
throw new NotImplementedException({ "Внутренний endpoint для Gateway. На L1 cache miss Backend проверяет PostgreSQL и S3, создаёт variant при необходимости, публикует RabbitMQ job, ждёт генерацию worker и возвращает готовые image bytes.",
message: "image read-through pipeline is not implemented yet",
request,
status: "not_implemented",
}) })
@ApiOkResponse({
description: "Variant уже был готов в S3 или был успешно сгенерирован worker.",
content: {
"image/*": {
schema: {
format: "binary",
type: "string",
},
},
},
})
@ApiBadRequestResponse({ description: "Некорректный assetId, version, preset, width, quality или format." })
@ApiNotFoundResponse({ description: "Asset или указанная версия source image не найдены." })
@ApiBadGatewayResponse({ description: "Worker/imgproxy/S3 не смогли подготовить или вернуть variant." })
@ApiResponse({ status: 504, description: "Variant не успел сгенерироваться до истечения IMAGE_ENSURE_WAIT_MS." })
@Header("Cache-Control", "public, max-age=31536000, immutable")
async ensureImageVariant(
@Body() request: EnsureImageVariantRequestDto,
@Res({ passthrough: true }) response: Response,
): Promise<StreamableFile> {
const result = await this.internalImages.ensureImageVariant(request)
response.setHeader("Cache-Control", result.cacheControl)
response.setHeader("Content-Length", result.contentLength.toString())
response.setHeader("Content-Type", result.contentType)
if (result.etag) {
response.setHeader("ETag", result.etag)
}
if (result.vary) {
response.setHeader("Vary", result.vary)
}
return new StreamableFile(result.body)
} }
} }

View File

@@ -0,0 +1,326 @@
import {
BadGatewayException,
BadRequestException,
GatewayTimeoutException,
Injectable,
NotFoundException,
} from "@nestjs/common"
import { imageAssetVersions, imageAssets, imageVariants } from "@image-platform/database"
import {
ImageTransformConfigError,
normalizeImageTransform,
parseBooleanFlag,
type ActualImageFormat,
type NormalizedImageTransform,
type RequestedImageFormat,
} from "@image-platform/image-config"
import { buildVariantImageKey } from "@image-platform/storage"
import { and, eq } from "drizzle-orm"
import { createHash } from "node:crypto"
import { DatabaseService } from "../infra/database.service"
import { QueueService } from "../infra/queue.service"
import { StorageService } from "../infra/storage.service"
import type { EnsureImageVariantRequestDto } from "./ensure-image-variant.dto"
type NormalizedEnsureRequest = NormalizedImageTransform & {
assetId: string
version: number
}
type VariantRow = typeof imageVariants.$inferSelect
export type EnsuredImageVariant = {
body: Buffer
cacheControl: string
contentLength: number
contentType: string
etag: string | null
vary: string | null
}
@Injectable()
export class InternalImagesService {
private readonly ensureWaitMs = parsePositiveInteger(process.env.IMAGE_ENSURE_WAIT_MS, 15_000)
constructor(
private readonly database: DatabaseService,
private readonly queue: QueueService,
private readonly storage: StorageService,
) {}
async ensureImageVariant(request: EnsureImageVariantRequestDto): Promise<EnsuredImageVariant> {
const normalized = normalizeRequest(request)
const assetVersion = await this.loadAssetVersion(normalized)
let variant = await this.findOrCreateVariant(normalized, assetVersion)
if (variant.status === "ready") {
const ready = await this.loadReadyObject(variant, normalized.requestedFormat)
if (ready) {
return ready
}
variant = await this.markVariantPending(variant.id)
}
if (variant.status === "failed") {
variant = await this.markVariantPending(variant.id)
}
this.queue.publishGenerateVariant(variant.id)
const readyVariant = await this.waitForReadyVariant(variant.id)
const ready = await this.loadReadyObject(readyVariant, normalized.requestedFormat)
if (!ready) {
throw new BadGatewayException("variant was marked ready but S3 object is missing")
}
return ready
}
private async loadAssetVersion(request: NormalizedEnsureRequest) {
const [row] = await this.database.db
.select({
assetId: imageAssets.id,
assetStatus: imageAssets.status,
assetVersionId: imageAssetVersions.id,
})
.from(imageAssets)
.innerJoin(imageAssetVersions, eq(imageAssetVersions.assetId, imageAssets.id))
.where(and(eq(imageAssets.publicId, request.assetId), eq(imageAssetVersions.version, request.version)))
.limit(1)
if (!row || row.assetStatus !== "active") {
throw new NotFoundException("image asset version not found")
}
return row
}
private async findOrCreateVariant(
request: NormalizedEnsureRequest,
assetVersion: { assetId: string; assetVersionId: string },
): Promise<VariantRow> {
const existing = await this.findVariant(request, assetVersion.assetId)
if (existing) {
return existing
}
const variantHash = createVariantHash(request)
const s3Key = buildVariantImageKey({
assetId: request.assetId,
format: request.format,
variantHash,
version: request.version,
})
const [created] = await this.database.db
.insert(imageVariants)
.values({
assetId: assetVersion.assetId,
assetVersion: request.version,
assetVersionId: assetVersion.assetVersionId,
format: request.format,
height: request.height,
preset: request.preset,
quality: request.quality,
requestedFormat: request.requestedFormat,
resizeMode: request.resize,
s3Key,
status: "pending",
variantHash,
width: request.width,
})
.onConflictDoNothing({
target: [
imageVariants.assetId,
imageVariants.assetVersion,
imageVariants.preset,
imageVariants.width,
imageVariants.height,
imageVariants.resizeMode,
imageVariants.quality,
imageVariants.format,
],
})
.returning()
if (created) {
return created
}
const raced = await this.findVariant(request, assetVersion.assetId)
if (!raced) {
throw new Error("failed to create image variant")
}
return raced
}
private async findVariant(request: NormalizedEnsureRequest, assetId: string) {
const [variant] = await this.database.db
.select()
.from(imageVariants)
.where(
and(
eq(imageVariants.assetId, assetId),
eq(imageVariants.assetVersion, request.version),
eq(imageVariants.preset, request.preset),
eq(imageVariants.width, request.width),
eq(imageVariants.height, request.height),
eq(imageVariants.resizeMode, request.resize),
eq(imageVariants.quality, request.quality),
eq(imageVariants.format, request.format),
),
)
.limit(1)
return variant ?? null
}
private async markVariantPending(variantId: string) {
const [variant] = await this.database.db
.update(imageVariants)
.set({ error: null, status: "pending", updatedAt: new Date() })
.where(eq(imageVariants.id, variantId))
.returning()
if (!variant) {
throw new NotFoundException("image variant not found")
}
return variant
}
private async loadReadyObject(
variant: VariantRow,
requestedFormat: RequestedImageFormat,
): Promise<EnsuredImageVariant | null> {
const object = await this.storage.getObject(variant.s3Key)
if (!object) {
return null
}
await this.database.db
.update(imageVariants)
.set({ lastAccessedAt: new Date(), updatedAt: new Date() })
.where(eq(imageVariants.id, variant.id))
return {
body: object.body,
cacheControl: "public, max-age=31536000, immutable",
contentLength: object.contentLength ?? object.body.length,
contentType: object.contentType ?? contentTypeForFormat(variant.format),
etag: object.etag,
vary: requestedFormat === "auto" ? "Accept" : null,
}
}
private async waitForReadyVariant(variantId: string) {
const deadline = Date.now() + this.ensureWaitMs
while (Date.now() <= deadline) {
const [variant] = await this.database.db.select().from(imageVariants).where(eq(imageVariants.id, variantId)).limit(1)
if (!variant) {
throw new NotFoundException("image variant not found")
}
if (variant.status === "ready") {
return variant
}
if (variant.status === "failed") {
throw new BadGatewayException(variant.error ?? "image variant generation failed")
}
await sleep(200)
}
throw new GatewayTimeoutException("image variant generation timed out")
}
}
function normalizeRequest(request: EnsureImageVariantRequestDto): NormalizedEnsureRequest {
if (!/^[a-zA-Z0-9_-]{3,128}$/.test(request.assetId)) {
throw new BadRequestException("assetId is invalid")
}
if (!isPositiveInteger(request.version)) {
throw new BadRequestException("version must be a positive integer")
}
try {
const transform = normalizeImageTransform({
allowCustomTransforms: parseBooleanFlag(process.env.IMAGE_ALLOW_CUSTOM_TRANSFORMS, false),
format: request.format,
height: request.height,
preset: request.preset,
quality: request.quality,
requestedFormat: request.requestedFormat,
resize: request.resize,
width: request.width,
})
return {
...transform,
assetId: request.assetId,
version: request.version,
}
} catch (error) {
if (error instanceof ImageTransformConfigError) {
throw new BadRequestException(error.message)
}
throw error
}
}
function createVariantHash(request: NormalizedEnsureRequest) {
return createHash("sha256")
.update(
[
request.assetId,
request.version,
request.preset,
request.width,
request.height,
request.resize,
request.quality,
request.format,
].join(":"),
)
.digest("hex")
.slice(0, 32)
}
function contentTypeForFormat(format: ActualImageFormat) {
if (format === "jpg") {
return "image/jpeg"
}
return `image/${format}`
}
function isPositiveInteger(value: number) {
return Number.isSafeInteger(value) && value > 0
}
function parsePositiveInteger(value: string | undefined, fallback: number) {
if (!value) {
return fallback
}
const parsed = Number.parseInt(value, 10)
return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback
}
function sleep(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms))
}

View File

@@ -11,13 +11,16 @@ async function bootstrap() {
const openApiConfig = new DocumentBuilder() const openApiConfig = new DocumentBuilder()
.setTitle("Image Platform API") .setTitle("Image Platform API")
.setDescription("Control plane for image assets, variants, S3 storage and external imgproxy.") .setDescription(
"Backend API для управления image assets, metadata в PostgreSQL, S3 variants, RabbitMQ jobs и генерацией через imgproxy.",
)
.setVersion("0.1.0") .setVersion("0.1.0")
.addTag("system") .addTag("system", "Системные endpoints для проверки состояния сервиса.")
.addTag("assets") .addTag("assets", "Регистрация и управление исходными изображениями.")
.addTag("variants") .addTag("variants", "Будущие endpoints для управления производными версиями изображений.")
.addTag("allowed-hosts") .addTag("allowed-hosts", "Будущие endpoints для управления разрешёнными source hosts.")
.addTag("internal-images") .addTag("internal-images", "Внутренние endpoints, которые вызывает Gateway на cache miss.")
.addTag("presets", "Статические presets, custom limits и mock allowlist source hosts.")
.build() .build()
const openApiDocument = SwaggerModule.createDocument(app, openApiConfig) const openApiDocument = SwaggerModule.createDocument(app, openApiConfig)
@@ -29,7 +32,7 @@ async function bootstrap() {
}, },
}) })
const port = Number.parseInt(process.env.API_PORT ?? "3001", 10) const port = Number.parseInt(process.env.BACKEND_PORT ?? process.env.API_PORT ?? "3001", 10)
await app.listen(port) await app.listen(port)
} }

View File

@@ -0,0 +1,49 @@
import { Controller, Get } from "@nestjs/common"
import { ApiOkResponse, ApiOperation, ApiTags } from "@nestjs/swagger"
import {
CUSTOM_TRANSFORM_CONFIG,
IMAGE_PRESETS,
loadAllowedSourceHostsFromEnv,
parseBooleanFlag,
type ImagePreset,
} from "@image-platform/image-config"
import { PresetsResponseDto } from "./presets.dto"
@ApiTags("presets")
@Controller("presets")
export class PresetsController {
@Get()
@ApiOperation({
summary: "получить доступные presets и custom config",
description: "Возвращает статический config presets, custom transform limits и mock allowlist source hosts.",
})
@ApiOkResponse({ description: "Конфигурация presets возвращена.", type: PresetsResponseDto })
getPresets(): PresetsResponseDto {
return {
allowedSourceHosts: [...loadAllowedSourceHostsFromEnv()],
custom: {
enabled: parseBooleanFlag(process.env.IMAGE_ALLOW_CUSTOM_TRANSFORMS, false),
formats: CUSTOM_TRANSFORM_CONFIG.formats,
maxHeight: CUSTOM_TRANSFORM_CONFIG.maxHeight,
maxWidth: CUSTOM_TRANSFORM_CONFIG.maxWidth,
quality: CUSTOM_TRANSFORM_CONFIG.quality,
},
presets: Object.entries(IMAGE_PRESETS).map(([name, preset]) => {
const config: ImagePreset = preset
return {
formats: config.formats,
height: config.height,
mode: config.mode,
name,
qualities: config.qualities,
quality: config.quality,
resize: config.resize,
width: config.width,
widths: config.widths,
}
}),
}
}
}

View File

@@ -0,0 +1,58 @@
import { ApiProperty } from "@nestjs/swagger"
export class PresetResponseDto {
@ApiProperty({ description: "Имя preset.", example: "card" })
name!: string
@ApiProperty({ description: "Режим preset.", enum: ["fixed", "responsive"], example: "responsive" })
mode!: string
@ApiProperty({ description: "Разрешённые форматы.", example: ["avif", "webp", "jpg"] })
formats!: readonly string[]
@ApiProperty({ description: "Разрешённые значения quality.", example: [75, 80] })
qualities!: readonly number[]
@ApiProperty({ description: "Quality по умолчанию.", example: 80 })
quality!: number
@ApiProperty({ description: "Resize mode preset.", enum: ["fit", "fill"], example: "fit" })
resize!: string
@ApiProperty({ description: "Фиксированная ширина для fixed preset.", example: 256, required: false })
width?: number
@ApiProperty({ description: "Фиксированная высота для fixed preset.", example: 256, required: false })
height?: number
@ApiProperty({ description: "Разрешённые ширины для responsive preset.", example: [320, 640, 960], required: false })
widths?: readonly number[]
}
export class CustomTransformConfigResponseDto {
@ApiProperty({ description: "Включены ли custom transforms.", example: true })
enabled!: boolean
@ApiProperty({ description: "Разрешённые форматы custom transforms.", example: ["avif", "webp", "jpg", "png"] })
formats!: readonly string[]
@ApiProperty({ description: "Максимальная ширина custom transform.", example: 4096 })
maxWidth!: number
@ApiProperty({ description: "Максимальная высота custom transform.", example: 4096 })
maxHeight!: number
@ApiProperty({ description: "Quality по умолчанию для custom transform.", example: 80 })
quality!: number
}
export class PresetsResponseDto {
@ApiProperty({ description: "Static presets.", type: [PresetResponseDto] })
presets!: PresetResponseDto[]
@ApiProperty({ description: "Custom transform config.", type: CustomTransformConfigResponseDto })
custom!: CustomTransformConfigResponseDto
@ApiProperty({ description: "Mock allowlist source hosts.", example: ["storage.yandexcloud.net"] })
allowedSourceHosts!: string[]
}

View File

@@ -5,11 +5,12 @@
"type": "module", "type": "module",
"scripts": { "scripts": {
"build": "tsc -p tsconfig.build.json", "build": "tsc -p tsconfig.build.json",
"dev": "tsx watch src/main.ts", "dev": "node --env-file-if-exists=../../.env ./node_modules/tsx/dist/cli.mjs watch src/main.ts",
"start": "node dist/main.js", "start": "node --env-file-if-exists=../../.env dist/main.js",
"typecheck": "tsc --noEmit -p tsconfig.json" "typecheck": "tsc --noEmit -p tsconfig.json"
}, },
"dependencies": { "dependencies": {
"@image-platform/image-config": "workspace:*",
"fastify": "^5.8.5" "fastify": "^5.8.5"
}, },
"devDependencies": { "devDependencies": {

View File

@@ -1,6 +1,8 @@
export type GatewayConfig = { export type GatewayConfig = {
backendUpstream: URL backendUpstream: URL
host: string host: string
l1MaxEntries: number
l1TtlMs: number
port: number port: number
} }
@@ -8,6 +10,8 @@ export function loadGatewayConfig(): GatewayConfig {
return { return {
backendUpstream: new URL(process.env.GATEWAY_BACKEND_UPSTREAM ?? "http://localhost:3001"), backendUpstream: new URL(process.env.GATEWAY_BACKEND_UPSTREAM ?? "http://localhost:3001"),
host: process.env.GATEWAY_HOST ?? "0.0.0.0", host: process.env.GATEWAY_HOST ?? "0.0.0.0",
l1MaxEntries: parsePositiveInteger(process.env.GATEWAY_L1_MAX_ENTRIES, 256),
l1TtlMs: parsePositiveInteger(process.env.GATEWAY_L1_TTL_MS, 10 * 60 * 1000),
port: parsePort(process.env.GATEWAY_PORT, 8888), port: parsePort(process.env.GATEWAY_PORT, 8888),
} }
} }
@@ -25,3 +29,13 @@ function parsePort(value: string | undefined, fallback: number) {
return parsed return parsed
} }
function parsePositiveInteger(value: string | undefined, fallback: number) {
if (!value) {
return fallback
}
const parsed = Number.parseInt(value, 10)
return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback
}

View File

@@ -0,0 +1,52 @@
export type CachedImage = {
body: Buffer
cacheControl: string
contentType: string
etag: string | null
}
export class ImageMemoryCache {
private readonly entries = new Map<string, CachedImage & { expiresAt: number }>()
constructor(
private readonly maxEntries: number,
private readonly ttlMs: number,
) {}
get(key: string): CachedImage | null {
const entry = this.entries.get(key)
if (!entry) {
return null
}
if (entry.expiresAt <= Date.now()) {
this.entries.delete(key)
return null
}
this.entries.delete(key)
this.entries.set(key, entry)
return {
body: entry.body,
cacheControl: entry.cacheControl,
contentType: entry.contentType,
etag: entry.etag,
}
}
set(key: string, image: CachedImage) {
this.entries.set(key, { ...image, expiresAt: Date.now() + this.ttlMs })
while (this.entries.size > this.maxEntries) {
const firstKey = this.entries.keys().next().value as string | undefined
if (!firstKey) {
break
}
this.entries.delete(firstKey)
}
}
}

View File

@@ -1,12 +1,23 @@
import Fastify from "fastify" import Fastify, { type FastifyReply } from "fastify"
import {
ImageTransformConfigError,
normalizeImageTransform,
parseBooleanFlag,
selectFormatForAccept,
type ActualImageFormat,
type ResizeMode,
} from "@image-platform/image-config"
import type { GatewayConfig } from "./config.js" import type { GatewayConfig } from "./config.js"
import { ImageMemoryCache, type CachedImage } from "./image-cache.js"
import { proxyToUpstream } from "./proxy.js" import { proxyToUpstream } from "./proxy.js"
export function createGatewayServer(config: GatewayConfig) { export function createGatewayServer(config: GatewayConfig) {
const app = Fastify({ const app = Fastify({
logger: true, logger: true,
}) })
const imageCache = new ImageMemoryCache(config.l1MaxEntries, config.l1TtlMs)
const allowCustomTransforms = parseBooleanFlag(process.env.IMAGE_ALLOW_CUSTOM_TRANSFORMS, false)
app.get("/health", async () => ({ app.get("/health", async () => ({
service: "image-platform-gateway", service: "image-platform-gateway",
@@ -26,19 +37,113 @@ export function createGatewayServer(config: GatewayConfig) {
} }
const width = parseOptionalInteger(request.query.w) const width = parseOptionalInteger(request.query.w)
const height = parseOptionalNonNegativeInteger(request.query.h)
const quality = parseOptionalInteger(request.query.q) const quality = parseOptionalInteger(request.query.q)
const format = request.query.f ?? "auto" const resize = parseResizeMode(request.query.fit)
return reply.code(501).header("cache-control", "no-store").send({ if (
assetId: request.params.assetId, (request.query.w !== undefined && width === null) ||
format, (request.query.h !== undefined && height === null) ||
message: "image gateway read-through pipeline is not implemented yet", (request.query.q !== undefined && quality === null)
) {
return reply.code(400).send({
message: "w, h and q query params must be positive integers",
statusCode: 400,
})
}
if (request.query.fit !== undefined && resize === null) {
return reply.code(400).send({
message: "fit query param must be fit or fill",
statusCode: 400,
})
}
const format = selectFormat({
acceptHeader: request.headers.accept,
allowCustomTransforms,
preset: request.params.preset,
requestedFormat: request.query.f ?? "auto",
})
if (!format.ok) {
return reply.code(400).send({
message: format.message,
statusCode: 400,
})
}
const transform = normalizeTransform({
allowCustomTransforms,
format: format.value.format,
height,
preset: request.params.preset, preset: request.params.preset,
quality, quality,
status: "not_implemented", requestedFormat: format.value.requestedFormat,
version, resize,
width, width,
}) })
if (!transform.ok) {
return reply.code(400).send({
message: transform.message,
statusCode: 400,
})
}
const cacheKey = buildImageCacheKey({
assetId: request.params.assetId,
format: transform.value.format,
height: transform.value.height,
preset: transform.value.preset,
quality: transform.value.quality,
resize: transform.value.resize,
version,
width: transform.value.width,
})
const cached = imageCache.get(cacheKey)
const vary = transform.value.requestedFormat === "auto" ? "Accept" : null
if (cached) {
return sendImage(reply, cached, "HIT", vary)
}
const backendResponse = await fetch(new URL("/api/internal/images/ensure", config.backendUpstream), {
body: JSON.stringify({
assetId: request.params.assetId,
format: transform.value.format,
height: transform.value.height,
preset: transform.value.preset,
quality: transform.value.quality,
requestedFormat: transform.value.requestedFormat,
resize: transform.value.resize,
version,
width: transform.value.width,
}),
headers: { "content-type": "application/json" },
method: "POST",
})
if (!backendResponse.ok) {
const body = await backendResponse.text()
return reply
.code(backendResponse.status)
.header("cache-control", "no-store")
.header("content-type", backendResponse.headers.get("content-type") ?? "text/plain; charset=utf-8")
.send(body)
}
const image: CachedImage = {
body: Buffer.from(await backendResponse.arrayBuffer()),
cacheControl: backendResponse.headers.get("cache-control") ?? "public, max-age=31536000, immutable",
contentType: backendResponse.headers.get("content-type") ?? contentTypeForFormat(transform.value.format),
etag: backendResponse.headers.get("etag"),
}
imageCache.set(cacheKey, image)
return sendImage(reply, image, "MISS", vary)
}, },
) )
@@ -59,10 +164,23 @@ export function createGatewayServer(config: GatewayConfig) {
type ImageQuery = { type ImageQuery = {
f?: string f?: string
fit?: string
h?: string
q?: string q?: string
w?: string w?: string
} }
type ImageCacheKeyInput = {
assetId: string
format: ActualImageFormat
height: number
preset: string
quality: number
resize: ResizeMode
version: number
width: number
}
type ImageParams = { type ImageParams = {
assetId: string assetId: string
preset: string preset: string
@@ -84,7 +202,96 @@ function parseOptionalInteger(value: string | undefined) {
return null return null
} }
if (!/^\d+$/.test(value)) {
return null
}
const parsed = Number.parseInt(value, 10) const parsed = Number.parseInt(value, 10)
return Number.isFinite(parsed) ? parsed : null return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : null
}
function parseOptionalNonNegativeInteger(value: string | undefined) {
if (!value) {
return null
}
if (!/^\d+$/.test(value)) {
return null
}
const parsed = Number.parseInt(value, 10)
return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : null
}
function parseResizeMode(value: string | undefined): ResizeMode | null {
if (value === undefined) {
return null
}
return value === "fit" || value === "fill" ? value : null
}
function selectFormat(input: Parameters<typeof selectFormatForAccept>[0]) {
try {
return { ok: true as const, value: selectFormatForAccept(input) }
} catch (error) {
if (error instanceof ImageTransformConfigError) {
return { message: error.message, ok: false as const }
}
throw error
}
}
function buildImageCacheKey(input: ImageCacheKeyInput) {
return [
input.assetId,
input.version,
input.preset,
input.width,
input.height,
input.resize,
input.quality,
input.format,
].join(":")
}
function normalizeTransform(input: Parameters<typeof normalizeImageTransform>[0]) {
try {
return { ok: true as const, value: normalizeImageTransform(input) }
} catch (error) {
if (error instanceof ImageTransformConfigError) {
return { message: error.message, ok: false as const }
}
throw error
}
}
function sendImage(reply: FastifyReply, image: CachedImage, cacheStatus: string, vary: string | null) {
if (image.etag) {
reply.header("etag", image.etag)
}
if (vary) {
reply.header("vary", vary)
}
return reply
.code(200)
.header("cache-control", image.cacheControl)
.header("content-length", image.body.length.toString())
.header("content-type", image.contentType)
.header("x-image-platform-l1", cacheStatus)
.send(image.body)
}
function contentTypeForFormat(format: ActualImageFormat) {
if (format === "jpg") {
return "image/jpeg"
}
return `image/${format}`
} }

View File

@@ -2,18 +2,19 @@
"name": "@image-platform/worker", "name": "@image-platform/worker",
"version": "0.1.0", "version": "0.1.0",
"private": true, "private": true,
"type": "module",
"scripts": { "scripts": {
"build": "tsc -p tsconfig.build.json", "build": "tsc -p tsconfig.build.json",
"dev": "tsx watch src/main.ts", "dev": "node --env-file-if-exists=../../.env ./node_modules/tsx/dist/cli.mjs watch src/main.ts",
"start": "node dist/main.js", "start": "node --env-file-if-exists=../../.env dist/main.js",
"typecheck": "tsc --noEmit -p tsconfig.json" "typecheck": "tsc --noEmit -p tsconfig.json"
}, },
"dependencies": { "dependencies": {
"@image-platform/database": "workspace:*", "@image-platform/database": "workspace:*",
"@image-platform/image-config": "workspace:*",
"@image-platform/queue": "workspace:*", "@image-platform/queue": "workspace:*",
"@image-platform/storage": "workspace:*", "@image-platform/storage": "workspace:*",
"amqplib": "^1.0.4" "amqplib": "^1.0.4",
"drizzle-orm": "^0.45.2"
}, },
"devDependencies": { "devDependencies": {
"@types/amqplib": "^0.10.8", "@types/amqplib": "^0.10.8",

View File

@@ -1,19 +1,34 @@
import { loadQueueTopologyFromEnv, type QueueTopology } from "@image-platform/queue" import { loadQueueTopologyFromEnv, type QueueTopology } from "@image-platform/queue"
import { loadStorageConfigFromEnv, type StorageConfig } from "@image-platform/storage"
export type WorkerConfig = { export type WorkerConfig = {
imgproxyUpstream: URL
prefetch: number prefetch: number
queueTopology: QueueTopology queueTopology: QueueTopology
rabbitmqUrl: string rabbitmqUrl: string
storage: StorageConfig
} }
export function loadWorkerConfig(env: NodeJS.ProcessEnv = process.env): WorkerConfig { export function loadWorkerConfig(env: NodeJS.ProcessEnv = process.env): WorkerConfig {
return { return {
imgproxyUpstream: new URL(getRequiredEnv(env, "IMGPROXY_UPSTREAM")),
prefetch: parsePositiveInteger(env.WORKER_PREFETCH, 2), prefetch: parsePositiveInteger(env.WORKER_PREFETCH, 2),
queueTopology: loadQueueTopologyFromEnv(env), queueTopology: loadQueueTopologyFromEnv(env),
rabbitmqUrl: env.RABBITMQ_URL ?? "amqp://image:image-password@localhost:5672/image_platform", rabbitmqUrl: getRequiredEnv(env, "RABBITMQ_URL"),
storage: loadStorageConfigFromEnv(env),
} }
} }
function getRequiredEnv(env: NodeJS.ProcessEnv, name: string) {
const value = env[name]
if (!value) {
throw new Error(`${name} is required`)
}
return value
}
function parsePositiveInteger(value: string | undefined, fallback: number) { function parsePositiveInteger(value: string | undefined, fallback: number) {
if (!value) { if (!value) {
return fallback return fallback

View File

@@ -1,10 +1,23 @@
import amqp, { type Channel, type ConsumeMessage } from "amqplib" import amqp, { type Channel, type ConsumeMessage } from "amqplib"
import { parseGenerateVariantJobBuffer, type QueueTopology } from "@image-platform/queue" import { and, eq, sql } from "drizzle-orm"
import {
createDatabase,
createDatabasePool,
imageAssets,
imageAssetVersions,
imageVariants,
} from "@image-platform/database"
import { assertQueueTopology, parseGenerateVariantJobBuffer } from "@image-platform/queue"
import type { ResizeMode } from "@image-platform/image-config"
import { createS3Client, putObjectBuffer } from "@image-platform/storage"
import { loadWorkerConfig } from "./config.js" import { loadWorkerConfig } from "./config.js"
async function bootstrap() { async function bootstrap() {
const config = loadWorkerConfig() const config = loadWorkerConfig()
const pool = createDatabasePool()
const db = createDatabase(pool)
const s3 = createS3Client(config.storage)
const connection = await amqp.connect(config.rabbitmqUrl) const connection = await amqp.connect(config.rabbitmqUrl)
const channel = await connection.createChannel() const channel = await connection.createChannel()
@@ -12,7 +25,7 @@ async function bootstrap() {
await channel.prefetch(config.prefetch) await channel.prefetch(config.prefetch)
await channel.consume( await channel.consume(
config.queueTopology.generateVariantQueue, config.queueTopology.generateVariantQueue,
(message) => void handleGenerateVariantMessage(channel, message), (message) => void handleGenerateVariantMessage({ channel, config, db, message, s3 }),
{ noAck: false }, { noAck: false },
) )
@@ -22,6 +35,7 @@ async function bootstrap() {
console.log("worker shutting down") console.log("worker shutting down")
await channel.close().catch((error: unknown) => console.error("failed to close RabbitMQ channel", error)) await channel.close().catch((error: unknown) => console.error("failed to close RabbitMQ channel", error))
await connection.close().catch((error: unknown) => console.error("failed to close RabbitMQ connection", error)) await connection.close().catch((error: unknown) => console.error("failed to close RabbitMQ connection", error))
await pool.end().catch((error: unknown) => console.error("failed to close PostgreSQL pool", error))
process.exit(0) process.exit(0)
} }
@@ -29,38 +43,179 @@ async function bootstrap() {
process.once("SIGTERM", () => void shutdown()) process.once("SIGTERM", () => void shutdown())
} }
async function assertQueueTopology(channel: Channel, topology: QueueTopology) { type GenerateVariantContext = {
await channel.assertExchange(topology.jobsExchange, "direct", { durable: true }) channel: Channel
await channel.assertExchange(topology.jobsDeadLetterExchange, "direct", { durable: true }) config: ReturnType<typeof loadWorkerConfig>
await channel.assertQueue(topology.generateVariantQueue, { db: ReturnType<typeof createDatabase>
deadLetterExchange: topology.jobsDeadLetterExchange, message: ConsumeMessage | null
deadLetterRoutingKey: topology.generateVariantDeadLetterRoutingKey, s3: ReturnType<typeof createS3Client>
durable: true,
})
await channel.assertQueue(topology.generateVariantDeadLetterQueue, { durable: true })
await channel.bindQueue(topology.generateVariantQueue, topology.jobsExchange, topology.generateVariantRoutingKey)
await channel.bindQueue(
topology.generateVariantDeadLetterQueue,
topology.jobsDeadLetterExchange,
topology.generateVariantDeadLetterRoutingKey,
)
} }
async function handleGenerateVariantMessage(channel: Channel, message: ConsumeMessage | null) { async function handleGenerateVariantMessage({ channel, config, db, message, s3 }: GenerateVariantContext) {
if (message === null) { if (message === null) {
return return
} }
try { try {
const job = parseGenerateVariantJobBuffer(message.content) const job = parseGenerateVariantJobBuffer(message.content)
console.log("generate variant job received, handler not implemented yet", job) const variant = await loadVariantForGeneration(db, job.variantId)
if (!variant) {
console.error("generate variant job references missing variant", job)
channel.nack(message, false, false) channel.nack(message, false, false)
return
}
if (variant.status === "ready") {
channel.ack(message)
return
}
await db
.update(imageVariants)
.set({
attemptCount: sql`${imageVariants.attemptCount} + 1`,
error: null,
status: "processing",
updatedAt: new Date(),
})
.where(and(eq(imageVariants.id, variant.id), eq(imageVariants.status, variant.status)))
const imgproxyUrl = buildImgproxyUrl(config.imgproxyUpstream, variant.sourceUrl, {
format: variant.format,
height: variant.height ?? 0,
quality: variant.quality,
resize: variant.resizeMode,
width: variant.width,
})
const response = await fetch(imgproxyUrl)
if (!response.ok) {
throw new Error(`imgproxy returned ${response.status} for variant ${variant.id}`)
}
const body = Buffer.from(await response.arrayBuffer())
const contentType = response.headers.get("content-type") ?? contentTypeForFormat(variant.format)
const putResult = await putObjectBuffer({
body,
bucket: config.storage.bucket,
cacheControl: "public, max-age=31536000, immutable",
client: s3,
contentType,
key: variant.s3Key,
})
await db
.update(imageVariants)
.set({
contentType,
error: null,
etag: putResult.ETag ?? null,
sizeBytes: body.length,
status: "ready",
updatedAt: new Date(),
})
.where(eq(imageVariants.id, variant.id))
console.log("generated image variant", { id: variant.id, key: variant.s3Key, sizeBytes: body.length })
channel.ack(message)
} catch (error) { } catch (error) {
console.error("invalid generate variant job", error) console.error("invalid generate variant job", error)
const variantId = getVariantIdFromMessage(message)
if (variantId) {
await db
.update(imageVariants)
.set({ error: formatError(error), status: "failed", updatedAt: new Date() })
.where(eq(imageVariants.id, variantId))
.catch((updateError: unknown) => console.error("failed to mark variant as failed", updateError))
}
channel.nack(message, false, false) channel.nack(message, false, false)
} }
} }
async function loadVariantForGeneration(db: ReturnType<typeof createDatabase>, variantId: string) {
const [row] = await db
.select({
format: imageVariants.format,
height: imageVariants.height,
id: imageVariants.id,
quality: imageVariants.quality,
resizeMode: imageVariants.resizeMode,
s3Key: imageVariants.s3Key,
sourceUrl: imageAssetVersions.sourceUrl,
status: imageVariants.status,
width: imageVariants.width,
})
.from(imageVariants)
.innerJoin(imageAssets, eq(imageVariants.assetId, imageAssets.id))
.innerJoin(imageAssetVersions, eq(imageVariants.assetVersionId, imageAssetVersions.id))
.where(eq(imageVariants.id, variantId))
.limit(1)
return row ?? null
}
function buildImgproxyUrl(
upstream: URL,
sourceUrl: string,
options: {
format: "avif" | "jpg" | "png" | "webp"
height: number
quality: number
resize: ResizeMode
width: number
},
) {
const url = new URL(upstream)
const encodedSource = Buffer.from(sourceUrl).toString("base64url")
url.pathname = joinUrlPath(
url.pathname,
"insecure",
`rs:${options.resize}:${options.width}:${options.height}`,
`q:${options.quality}`,
`${encodedSource}.${options.format}`,
)
return url
}
function joinUrlPath(...segments: string[]) {
return segments
.flatMap((segment) => segment.split("/"))
.filter(Boolean)
.map(encodePathSegment)
.join("/")
.replace(/^/, "/")
}
function encodePathSegment(segment: string) {
return segment.includes(":") ? segment : encodeURIComponent(segment)
}
function contentTypeForFormat(format: "avif" | "jpg" | "png" | "webp") {
if (format === "jpg") {
return "image/jpeg"
}
return `image/${format}`
}
function getVariantIdFromMessage(message: ConsumeMessage) {
try {
return parseGenerateVariantJobBuffer(message.content).variantId
} catch {
return null
}
}
function formatError(error: unknown) {
const message = error instanceof Error ? error.message : String(error)
return message.slice(0, 2000)
}
void bootstrap().catch((error: unknown) => { void bootstrap().catch((error: unknown) => {
console.error("worker failed to start", error) console.error("worker failed to start", error)
process.exit(1) process.exit(1)

View File

@@ -5,8 +5,8 @@
"esModuleInterop": true, "esModuleInterop": true,
"forceConsistentCasingInFileNames": true, "forceConsistentCasingInFileNames": true,
"lib": ["ES2023"], "lib": ["ES2023"],
"module": "NodeNext", "module": "Node16",
"moduleResolution": "NodeNext", "moduleResolution": "Node16",
"noUncheckedIndexedAccess": true, "noUncheckedIndexedAccess": true,
"outDir": "./dist", "outDir": "./dist",
"rootDir": "./src", "rootDir": "./src",

View File

@@ -1,6 +1,6 @@
# Черновик Backend Контракта # Черновик Backend Контракта
Это черновик будущего бизнес-контракта для NestJS backend. Сейчас реализованы system endpoints, Swagger и placeholder для internal image ensure. Это черновик бизнес-контракта для NestJS backend. Сейчас реализованы system endpoints, Swagger, `POST /api/assets` и internal image ensure MVP.
## System ## System
@@ -20,7 +20,7 @@ NestJS backend отдаёт JSON, metadata, statuses и URLs. Public image origi
POST /api/internal/images/ensure POST /api/internal/images/ensure
``` ```
Текущий статус реализации - endpoint зарезервирован и возвращает `501 Not Implemented`. Gateway `/images/*` пока тоже возвращает placeholder `501 Not Implemented`. Текущий статус реализации - MVP реализован: Backend создаёт/переиспользует variant row, публикует RabbitMQ job, ждёт `ready` до timeout и возвращает image bytes из S3.
Request body: Request body:
@@ -30,7 +30,10 @@ Request body:
"version": 4, "version": 4,
"preset": "card", "preset": "card",
"width": 640, "width": 640,
"height": 0,
"quality": 80, "quality": 80,
"requestedFormat": "auto",
"resize": "fit",
"format": "webp" "format": "webp"
} }
``` ```
@@ -39,13 +42,16 @@ Query params:
| Param | Описание | | Param | Описание |
|---|---| |---|---|
| `w` | целевая ширина, должна быть разрешена preset или округлена до ближайшей разрешённой | | `w` | целевая ширина; обязательна для responsive presets и custom, запрещена/фиксирована для fixed presets |
| `q` | качество, должно быть из allowlist качества | | `h` | целевая высота; используется для custom, для fixed preset берётся из config |
| `q` | качество; если не передано, берётся из preset, иначе должно входить в allowlist preset |
| `f` | `auto`, `avif`, `webp`, `jpg`; для Next/image по умолчанию `auto` | | `f` | `auto`, `avif`, `webp`, `jpg`; для Next/image по умолчанию `auto` |
| `fit` | `fit` или `fill`; используется только для custom transforms |
Responsibilities: Responsibilities:
- проверить `assetId` и `preset`; - проверить `assetId` и статический preset/custom transform config;
- нормализовать width, height, resize, quality и format;
- вычислить deterministic `variantHash`; - вычислить deterministic `variantHash`;
- проверить PostgreSQL и S3; - проверить PostgreSQL и S3;
- если variant готов в S3, вернуть bytes или stream metadata для Gateway; - если variant готов в S3, вернуть bytes или stream metadata для Gateway;
@@ -69,9 +75,8 @@ ETag: "..."
|---|---| |---|---|
| `400` | некорректные query params | | `400` | некорректные query params |
| `404` | asset или preset не найден | | `404` | asset или preset не найден |
| `409` | variant уже генерируется и sync ожидание отключено | | `502` | external imgproxy недоступен или S3 object отсутствует после `ready` |
| `422` | source image нельзя обработать | | `504` | generation не завершилась до `IMAGE_ENSURE_WAIT_MS` |
| `502` | external imgproxy недоступен |
Public endpoint реализуется в Fastify Gateway, internal ensure endpoint - в Backend: Public endpoint реализуется в Fastify Gateway, internal ensure endpoint - в Backend:
@@ -82,6 +87,14 @@ client -> CDN -> Gateway /images/* -> Backend ensure -> RabbitMQ -> Worker -> im
Важно: `/images/*` не должен жить в NestJS `/api`. Gateway остаётся public image origin, Backend остаётся управлением процессов. Важно: `/images/*` не должен жить в NestJS `/api`. Gateway остаётся public image origin, Backend остаётся управлением процессов.
Public image URL versioned: `/images/{assetId}/v{version}/{preset}?w={width}&q={quality}&f=auto`. Public image URL versioned: `/images/{assetId}/v{version}/{preset}?w={width}&q={quality}&f=auto`.
Примеры:
```text
/images/asset_demo/v1/card?w=640&q=80&f=auto
/images/asset_demo/v1/avatar?f=auto
/images/asset_demo/v1/custom?w=777&h=333&q=72&fit=fill&f=webp
```
## Allowed Hosts ## Allowed Hosts
```text ```text
@@ -94,63 +107,152 @@ DELETE /allowed-hosts/:id
## Assets ## Assets
```text ```text
GET /assets
POST /assets POST /assets
GET /assets/:id GET /assets
GET /assets/:publicId
GET /assets/:publicId/variants
POST /assets/:publicId/variants
DELETE /assets/:id DELETE /assets/:id
``` ```
Сейчас реализованы `POST /assets`, `GET /assets`, `GET /assets/:publicId`, `GET /assets/:publicId/variants`, `POST /assets/:publicId/variants`.
`POST /assets` request: `POST /assets` request:
```json ```json
{ {
"sourceUrl": "https://example.com/photo.jpg" "sourceUrl": "https://storage.yandexcloud.net/shared1318/img/1.jpg",
"publicId": "asset_demo"
}
```
Response:
```json
{
"id": "59fcf4f6-9891-4df4-8bb7-a4dbe570bb66",
"publicId": "asset_demo",
"version": 1,
"sourceHost": "storage.yandexcloud.net",
"imageBasePath": "/images/asset_demo/v1/card"
} }
``` ```
Responsibilities: Responsibilities:
- validate source URL; - validate source URL;
- check `allowed_image_hosts`; - check mock allowlist `SOURCE_ALLOWED_HOSTS`, если `SOURCE_HOST_ALLOW_ALL=false`;
- create or reuse `image_assets` row; - создать `image_assets` row;
- создать `image_asset_versions` row версии `1`;
- optionally save original to S3 later. - optionally save original to S3 later.
`GET /assets` response:
```json
{
"assets": [
{
"id": "59fcf4f6-9891-4df4-8bb7-a4dbe570bb66",
"publicId": "asset_demo",
"currentVersion": 1,
"status": "active",
"sourceUrl": "https://storage.yandexcloud.net/shared1318/img/1.jpg",
"sourceHost": "storage.yandexcloud.net",
"createdAt": "2026-05-05T12:00:00.000Z",
"updatedAt": "2026-05-05T12:00:00.000Z"
}
]
}
```
`GET /assets/:publicId/variants` возвращает rows из `image_variants` с public Gateway URL, S3 key и status.
## Variants ## Variants
```text ```text
GET /assets/:id/variants GET /assets/:publicId/variants
POST /assets/:id/variants POST /assets/:publicId/variants
POST /variants/:id/regenerate POST /variants/:id/regenerate
DELETE /variants/:id DELETE /variants/:id
``` ```
`POST /assets/:id/variants` request: `POST /assets/:publicId/variants` request:
```json ```json
{ {
"preset": "card", "preset": "card",
"mode": "single",
"format": "webp", "format": "webp",
"width": 640 "width": 640,
"quality": 80
} }
``` ```
Response if ready: Family generation:
```json ```json
{ {
"id": "variant_123", "preset": "card",
"status": "ready", "mode": "family"
"url": "http://localhost:8888/images/asset_123/v4/card?w=640&q=80&f=webp"
} }
``` ```
Response if generation is async: Для `family` Backend создаёт набор variants по разрешённым widths/formats static preset и ставит RabbitMQ jobs. Endpoint не ждёт image bytes, в отличие от internal ensure.
Response:
```json ```json
{ {
"publicId": "asset_demo",
"version": 1,
"variants": [
{
"id": "variant_123", "id": "variant_123",
"preset": "card",
"version": 1,
"width": 640,
"height": 0,
"resize": "fit",
"quality": 80,
"requestedFormat": "webp",
"format": "webp",
"status": "pending", "status": "pending",
"url": null "url": "http://localhost:8888/images/asset_demo/v1/card?w=640&q=80&f=webp",
"s3Key": "variants/asset_demo/v1/abc.webp"
}
]
}
```
## Presets
```text
GET /presets
```
Возвращает static presets, custom transform limits и mock allowlist source hosts.
```json
{
"presets": [
{
"name": "card",
"mode": "responsive",
"formats": ["avif", "webp", "jpg"],
"qualities": [75, 80],
"quality": 80,
"resize": "fit",
"widths": [320, 640, 960]
}
],
"custom": {
"enabled": true,
"formats": ["avif", "webp", "jpg", "png"],
"maxWidth": 4096,
"maxHeight": 4096,
"quality": 80
},
"allowedSourceHosts": ["storage.yandexcloud.net"]
} }
``` ```

View File

@@ -68,6 +68,7 @@ requested_format requested_format not null default auto
format variant_format not null format variant_format not null
width integer not null width integer not null
height integer nullable height integer nullable
resize_mode resize_mode not null default fit
quality integer not null quality integer not null
s3_key text not null unique s3_key text not null unique
content_type text nullable content_type text nullable
@@ -89,6 +90,7 @@ updated_at timestamptz not null default now()
asset_status: active | disabled | deleted asset_status: active | disabled | deleted
requested_format: auto | avif | webp | jpg | png requested_format: auto | avif | webp | jpg | png
variant_format: avif | webp | jpg | png variant_format: avif | webp | jpg | png
resize_mode: fit | fill
variant_status: pending | processing | ready | failed variant_status: pending | processing | ready | failed
``` ```
@@ -98,7 +100,7 @@ variant_status: pending | processing | ready | failed
allowed_image_hosts(hostname) allowed_image_hosts(hostname)
image_assets(public_id) image_assets(public_id)
image_asset_versions(asset_id, version) image_asset_versions(asset_id, version)
image_variants(asset_id, asset_version, preset, width, quality, format) image_variants(asset_id, asset_version, preset, width, height, resize_mode, quality, format)
image_variants(s3_key) image_variants(s3_key)
image_variants(variant_hash) image_variants(variant_hash)
``` ```
@@ -123,6 +125,8 @@ variants/{assetId}/v{version}/{variantHash}.{format}
- `assetVersion`; - `assetVersion`;
- `preset`; - `preset`;
- normalized width; - normalized width;
- normalized height, где `0` означает auto height;
- normalized resize mode;
- normalized quality; - normalized quality;
- фактический output format; - фактический output format;
- параметры transform, влияющие на bytes. - параметры transform, влияющие на bytes.
@@ -135,34 +139,45 @@ variants/asset_123/v4/card_w640_q80_webp.webp
variants/asset_123/v4/card_w640_q80_jpg.jpg variants/asset_123/v4/card_w640_q80_jpg.jpg
``` ```
Public URL также versioned: Public URL также versioned. Для fixed preset `w` и `q` можно не передавать, для responsive preset `w` обязателен:
```text ```text
/images/{assetId}/v{version}/{preset}?w={width}&q={quality}&f=auto /images/{assetId}/v{version}/{preset}?w={width}&q={quality}&f=auto
/images/{assetId}/v{version}/avatar?f=auto
``` ```
## Presets ## Presets
Клиент не должен передавать произвольные трансформации. Сначала нужны ограниченные presets. Клиент не должен бесконтрольно создавать произвольные трансформации. Сейчас есть статический config в `packages/image-config`.
Режимы:
- `fixed` - preset задаёт один размер, например `avatar`.
- `responsive` - preset задаёт allowlist ширин, например `card` и `hero`.
- `custom` - произвольный single image, только если включён `IMAGE_ALLOW_CUSTOM_TRANSFORMS=true`.
Пример: Пример:
```text ```text
avatar: avatar:
widths: 128, 256, 512 mode: fixed
width: 256
height: 256
formats: avif, webp, jpg formats: avif, webp, jpg
quality: 80 quality: 80
resize: fill resize: fill
card: card:
mode: responsive
widths: 320, 640, 960 widths: 320, 640, 960
formats: avif, webp, jpg formats: avif, webp, jpg
quality: 80 qualities: 75, 80
resize: fit resize: fit
hero: hero:
mode: responsive
widths: 1280, 1920 widths: 1280, 1920
formats: avif, webp, jpg formats: avif, webp, jpg
quality: 80 qualities: 75, 80
resize: fit resize: fit
``` ```

View File

@@ -12,7 +12,7 @@
- imgproxy dev instance; - imgproxy dev instance;
- RabbitMQ. - RabbitMQ.
`backend` уже создан как NestJS app. `admin` уже создан как чистый Vite React TS app. `gateway` уже создан как Fastify app. `worker` уже создан как RabbitMQ skeleton. `backend` уже умеет регистрировать assets и выполнять internal ensure. `gateway` уже ходит в Backend и держит L1 memory cache. `worker` уже читает RabbitMQ jobs, вызывает imgproxy и пишет variants в S3.
Gateway обязателен для Cloudinary-like поведения и интеграции с `next/image`. Gateway обязателен для Cloudinary-like поведения и интеграции с `next/image`.
## Запуск инфраструктуры ## Запуск инфраструктуры
@@ -21,8 +21,11 @@ Gateway обязателен для Cloudinary-like поведения и инт
cp .env.example .env cp .env.example .env
pnpm install pnpm install
pnpm infra:up pnpm infra:up
pnpm db:migrate
``` ```
`.env` используется только локально и игнорируется git. Backend, Gateway, Worker и Drizzle scripts автоматически подхватывают его через Node `--env-file-if-exists`; в production эти переменные должны приходить из окружения процесса.
Проверить compose config: Проверить compose config:
```bash ```bash
@@ -73,6 +76,31 @@ curl http://localhost:3001/api/health
open http://localhost:3001/docs open http://localhost:3001/docs
``` ```
Зарегистрировать source image в dev mode:
```bash
curl -sS -X POST http://localhost:3001/api/assets \
-H 'content-type: application/json' \
-d '{"sourceUrl":"https://storage.yandexcloud.net/shared1318/img/1.jpg","publicId":"asset_demo"}'
```
Посмотреть business API:
```bash
curl -sS http://localhost:3001/api/presets
curl -sS http://localhost:3001/api/assets
curl -sS http://localhost:3001/api/assets/asset_demo
curl -sS http://localhost:3001/api/assets/asset_demo/variants
```
Явно поставить jobs на генерацию family variants без Gateway lazy request:
```bash
curl -sS -X POST http://localhost:3001/api/assets/asset_demo/variants \
-H 'content-type: application/json' \
-d '{"preset":"card","mode":"family"}'
```
## Database ## Database
Drizzle schema живёт в `packages/database/src/schema.ts`, миграции - в `packages/database/drizzle`. Drizzle schema живёт в `packages/database/src/schema.ts`, миграции - в `packages/database/drizzle`.
@@ -130,17 +158,27 @@ pnpm gateway:dev
curl http://localhost:8888/health curl http://localhost:8888/health
``` ```
Проверить placeholder image origin route: Проверить image origin route после запуска Backend и Worker:
```bash ```bash
curl -i "http://localhost:8888/images/asset_123/v4/card?w=640&q=80&f=auto" curl -i "http://localhost:8888/images/asset_demo/v1/card?w=640&q=80&f=auto"
curl -i "http://localhost:8888/images/asset_demo/v1/avatar?f=auto"
curl -i "http://localhost:8888/images/asset_demo/v1/custom?w=777&h=333&q=72&fit=fill&f=webp"
``` ```
Ожидаемый текущий статус - `501`, потому что S3/imgproxy генерация ещё не реализована. Первый запрос должен пройти через Backend/RabbitMQ/Worker/imgproxy/S3 и вернуться с `x-image-platform-l1: MISS`. Повторный запрос должен вернуться из gateway L1 с `x-image-platform-l1: HIT`.
Статические presets сейчас лежат в `packages/image-config`:
- `card` - responsive, widths `320`, `640`, `960`.
- `hero` - responsive, widths `1280`, `1920`.
- `avatar` - fixed `256x256`.
Mock allowlist source hosts задаётся через `SOURCE_ALLOWED_HOSTS`. В dev по умолчанию разрешён `storage.yandexcloud.net`.
## Worker ## Worker
Worker запускается нодой и сейчас только объявляет RabbitMQ topology, слушает `image.generate-variant` и отправляет полученные jobs в DLQ, потому что генерация ещё не реализована. Worker запускается нодой, объявляет RabbitMQ topology, слушает `image.generate-variant`, вызывает `imgproxy` и пишет готовый variant в S3.
```bash ```bash
pnpm worker:dev pnpm worker:dev

View File

@@ -18,7 +18,7 @@ IMGPROXY_SALT=
Пример path: Пример path:
```text ```text
/unsafe/resize:fit:800:0:0/q:80/plain/https://example.com/photo.jpg /unsafe/resize:fit:800:0:0/q:80/plain/https://storage.yandexcloud.net/shared1318/img/1.jpg
``` ```
## Prod режим ## Prod режим
@@ -28,7 +28,7 @@ IMGPROXY_SALT=
Path для подписи строится без `/unsafe`: Path для подписи строится без `/unsafe`:
```text ```text
/resize:fit:800:0:0/q:80/plain/https://example.com/photo.jpg /resize:fit:800:0:0/q:80/plain/https://storage.yandexcloud.net/shared1318/img/1.jpg
``` ```
Signature: Signature:
@@ -66,6 +66,6 @@ Final signed URL:
- Source URL валидировать в Backend/worker. - Source URL валидировать в Backend/worker.
- Разрешать только `http` и `https`. - Разрешать только `http` и `https`.
- Запрещать localhost, private IP, loopback, link-local. - Запрещать localhost, private IP, loopback, link-local.
- Source host должен быть enabled в `allowed_image_hosts`. - Source host должен быть разрешён mock allowlist `SOURCE_ALLOWED_HOSTS`; таблица `allowed_image_hosts` остаётся для будущего CRUD.
- Не давать клиенту произвольные imgproxy options. - Не давать клиенту произвольные imgproxy options без `IMAGE_ALLOW_CUSTOM_TRANSFORMS=true`.
- Использовать presets и deterministic `variantHash`. - Использовать static presets/custom normalization и deterministic `variantHash`.

View File

@@ -22,7 +22,7 @@ module.exports = {
images: { images: {
loader: "custom", loader: "custom",
loaderFile: "./src/image-platform-loader.js", loaderFile: "./src/image-platform-loader.js",
qualities: [60, 75, 80, 90], qualities: [75, 80],
}, },
} }
``` ```
@@ -58,7 +58,7 @@ export function ProductCard() {
GET /images/{assetId}/v{version}/{preset}?w={width}&q={quality}&f=auto GET /images/{assetId}/v{version}/{preset}?w={width}&q={quality}&f=auto
``` ```
Сейчас route уже зарезервирован в Fastify Gateway, но возвращает placeholder `501`, пока не реализованы PostgreSQL/S3/imgproxy read-through операции. Route реализован в Fastify Gateway. Для `card` ширина должна входить в static preset allowlist: `320`, `640`, `960`.
Пример: Пример:

View File

@@ -13,31 +13,33 @@
"admin:dev": "pnpm --filter @image-platform/admin dev", "admin:dev": "pnpm --filter @image-platform/admin dev",
"admin:preview": "pnpm --filter @image-platform/admin preview", "admin:preview": "pnpm --filter @image-platform/admin preview",
"admin:typecheck": "pnpm --filter @image-platform/admin typecheck", "admin:typecheck": "pnpm --filter @image-platform/admin typecheck",
"backend:build": "pnpm --filter @image-platform/backend build", "backend:build": "pnpm image-config:build && pnpm db:build && pnpm queue:build && pnpm storage:build && pnpm --filter @image-platform/backend build",
"backend:dev": "pnpm --filter @image-platform/backend dev", "backend:dev": "pnpm image-config:build && pnpm db:build && pnpm queue:build && pnpm storage:build && pnpm --filter @image-platform/backend dev",
"backend:start": "pnpm --filter @image-platform/backend start", "backend:start": "pnpm image-config:build && pnpm db:build && pnpm queue:build && pnpm storage:build && pnpm --filter @image-platform/backend start",
"backend:typecheck": "pnpm --filter @image-platform/backend typecheck", "backend:typecheck": "pnpm --filter @image-platform/backend typecheck",
"db:build": "pnpm --filter @image-platform/database build", "db:build": "pnpm --filter @image-platform/database build",
"db:generate": "pnpm --filter @image-platform/database db:generate", "db:generate": "pnpm --filter @image-platform/database db:generate",
"db:migrate": "pnpm --filter @image-platform/database db:migrate", "db:migrate": "pnpm --filter @image-platform/database db:migrate",
"db:studio": "pnpm --filter @image-platform/database db:studio", "db:studio": "pnpm --filter @image-platform/database db:studio",
"db:typecheck": "pnpm --filter @image-platform/database typecheck", "db:typecheck": "pnpm --filter @image-platform/database typecheck",
"gateway:build": "pnpm --filter @image-platform/gateway build", "gateway:build": "pnpm image-config:build && pnpm --filter @image-platform/gateway build",
"gateway:dev": "pnpm --filter @image-platform/gateway dev", "gateway:dev": "pnpm image-config:build && pnpm --filter @image-platform/gateway dev",
"gateway:start": "pnpm --filter @image-platform/gateway start", "gateway:start": "pnpm image-config:build && pnpm --filter @image-platform/gateway start",
"gateway:typecheck": "pnpm --filter @image-platform/gateway typecheck", "gateway:typecheck": "pnpm --filter @image-platform/gateway typecheck",
"image-config:build": "pnpm --filter @image-platform/image-config build",
"image-config:typecheck": "pnpm --filter @image-platform/image-config typecheck",
"queue:build": "pnpm --filter @image-platform/queue build", "queue:build": "pnpm --filter @image-platform/queue build",
"queue:typecheck": "pnpm --filter @image-platform/queue typecheck", "queue:typecheck": "pnpm --filter @image-platform/queue typecheck",
"storage:build": "pnpm --filter @image-platform/storage build", "storage:build": "pnpm --filter @image-platform/storage build",
"storage:typecheck": "pnpm --filter @image-platform/storage typecheck", "storage:typecheck": "pnpm --filter @image-platform/storage typecheck",
"worker:build": "pnpm db:build && pnpm queue:build && pnpm storage:build && pnpm --filter @image-platform/worker build", "worker:build": "pnpm image-config:build && pnpm db:build && pnpm queue:build && pnpm storage:build && pnpm --filter @image-platform/worker build",
"worker:dev": "pnpm db:build && pnpm queue:build && pnpm storage:build && pnpm --filter @image-platform/worker dev", "worker:dev": "pnpm image-config:build && pnpm db:build && pnpm queue:build && pnpm storage:build && pnpm --filter @image-platform/worker dev",
"worker:start": "pnpm --filter @image-platform/worker start", "worker:start": "pnpm image-config:build && pnpm --filter @image-platform/worker start",
"worker:typecheck": "pnpm --filter @image-platform/worker typecheck", "worker:typecheck": "pnpm --filter @image-platform/worker typecheck",
"infra:config": "docker compose -f infra/compose.dev.yml config", "infra:config": "docker compose -f infra/compose.dev.yml config",
"infra:up": "docker compose -f infra/compose.dev.yml up -d", "infra:up": "docker compose -f infra/compose.dev.yml up -d",
"infra:down": "docker compose -f infra/compose.dev.yml down", "infra:down": "docker compose -f infra/compose.dev.yml down",
"infra:logs": "docker compose -f infra/compose.dev.yml logs -f", "infra:logs": "docker compose -f infra/compose.dev.yml logs -f",
"check": "pnpm infra:config && pnpm db:typecheck && pnpm queue:typecheck && pnpm storage:typecheck && pnpm backend:typecheck && pnpm admin:typecheck && pnpm gateway:typecheck && pnpm worker:typecheck" "check": "pnpm infra:config && pnpm image-config:typecheck && pnpm db:typecheck && pnpm queue:typecheck && pnpm storage:typecheck && pnpm backend:typecheck && pnpm admin:typecheck && pnpm gateway:typecheck && pnpm worker:typecheck"
} }
} }

View File

@@ -1,8 +1,14 @@
import { defineConfig } from "drizzle-kit" import { defineConfig } from "drizzle-kit"
const databaseUrl = process.env.DATABASE_URL
if (!databaseUrl) {
throw new Error("DATABASE_URL is required")
}
export default defineConfig({ export default defineConfig({
dbCredentials: { dbCredentials: {
url: process.env.DATABASE_URL ?? "postgres://image:image-password@localhost:5433/image_platform", url: databaseUrl,
}, },
dialect: "postgresql", dialect: "postgresql",
out: "./drizzle", out: "./drizzle",

View File

@@ -0,0 +1,4 @@
CREATE TYPE "public"."resize_mode" AS ENUM('fit', 'fill');--> statement-breakpoint
DROP INDEX "image_variants_lookup_idx";--> statement-breakpoint
ALTER TABLE "image_variants" ADD COLUMN "resize_mode" "resize_mode" DEFAULT 'fit' NOT NULL;--> statement-breakpoint
CREATE UNIQUE INDEX "image_variants_lookup_idx" ON "image_variants" USING btree ("asset_id","asset_version","preset","width","height","resize_mode","quality","format");

View File

@@ -0,0 +1,632 @@
{
"id": "9b706710-b809-4324-8632-634884f75166",
"prevId": "72292622-d326-46fe-8e6a-90096c7e6634",
"version": "7",
"dialect": "postgresql",
"tables": {
"public.allowed_image_hosts": {
"name": "allowed_image_hosts",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "uuid",
"primaryKey": true,
"notNull": true,
"default": "gen_random_uuid()"
},
"hostname": {
"name": "hostname",
"type": "text",
"primaryKey": false,
"notNull": true
},
"enabled": {
"name": "enabled",
"type": "boolean",
"primaryKey": false,
"notNull": true,
"default": true
},
"description": {
"name": "description",
"type": "text",
"primaryKey": false,
"notNull": false
},
"created_at": {
"name": "created_at",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": true,
"default": "now()"
},
"updated_at": {
"name": "updated_at",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": true,
"default": "now()"
}
},
"indexes": {
"allowed_image_hosts_hostname_idx": {
"name": "allowed_image_hosts_hostname_idx",
"columns": [
{
"expression": "hostname",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": true,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"policies": {},
"checkConstraints": {},
"isRLSEnabled": false
},
"public.image_asset_versions": {
"name": "image_asset_versions",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "uuid",
"primaryKey": true,
"notNull": true,
"default": "gen_random_uuid()"
},
"asset_id": {
"name": "asset_id",
"type": "uuid",
"primaryKey": false,
"notNull": true
},
"version": {
"name": "version",
"type": "integer",
"primaryKey": false,
"notNull": true
},
"source_url": {
"name": "source_url",
"type": "text",
"primaryKey": false,
"notNull": true
},
"source_host": {
"name": "source_host",
"type": "text",
"primaryKey": false,
"notNull": true
},
"source_hash": {
"name": "source_hash",
"type": "text",
"primaryKey": false,
"notNull": true
},
"original_s3_key": {
"name": "original_s3_key",
"type": "text",
"primaryKey": false,
"notNull": false
},
"width": {
"name": "width",
"type": "integer",
"primaryKey": false,
"notNull": false
},
"height": {
"name": "height",
"type": "integer",
"primaryKey": false,
"notNull": false
},
"content_type": {
"name": "content_type",
"type": "text",
"primaryKey": false,
"notNull": false
},
"size_bytes": {
"name": "size_bytes",
"type": "bigint",
"primaryKey": false,
"notNull": false
},
"created_at": {
"name": "created_at",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": true,
"default": "now()"
}
},
"indexes": {
"image_asset_versions_asset_version_idx": {
"name": "image_asset_versions_asset_version_idx",
"columns": [
{
"expression": "asset_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "version",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": true,
"concurrently": false,
"method": "btree",
"with": {}
},
"image_asset_versions_source_hash_idx": {
"name": "image_asset_versions_source_hash_idx",
"columns": [
{
"expression": "source_hash",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {
"image_asset_versions_asset_id_image_assets_id_fk": {
"name": "image_asset_versions_asset_id_image_assets_id_fk",
"tableFrom": "image_asset_versions",
"tableTo": "image_assets",
"columnsFrom": [
"asset_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"policies": {},
"checkConstraints": {},
"isRLSEnabled": false
},
"public.image_assets": {
"name": "image_assets",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "uuid",
"primaryKey": true,
"notNull": true,
"default": "gen_random_uuid()"
},
"public_id": {
"name": "public_id",
"type": "text",
"primaryKey": false,
"notNull": true
},
"current_version": {
"name": "current_version",
"type": "integer",
"primaryKey": false,
"notNull": true,
"default": 1
},
"status": {
"name": "status",
"type": "asset_status",
"typeSchema": "public",
"primaryKey": false,
"notNull": true,
"default": "'active'"
},
"created_at": {
"name": "created_at",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": true,
"default": "now()"
},
"updated_at": {
"name": "updated_at",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": true,
"default": "now()"
}
},
"indexes": {
"image_assets_public_id_idx": {
"name": "image_assets_public_id_idx",
"columns": [
{
"expression": "public_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": true,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"policies": {},
"checkConstraints": {},
"isRLSEnabled": false
},
"public.image_variants": {
"name": "image_variants",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "uuid",
"primaryKey": true,
"notNull": true,
"default": "gen_random_uuid()"
},
"asset_id": {
"name": "asset_id",
"type": "uuid",
"primaryKey": false,
"notNull": true
},
"asset_version_id": {
"name": "asset_version_id",
"type": "uuid",
"primaryKey": false,
"notNull": true
},
"asset_version": {
"name": "asset_version",
"type": "integer",
"primaryKey": false,
"notNull": true
},
"preset": {
"name": "preset",
"type": "text",
"primaryKey": false,
"notNull": true
},
"variant_hash": {
"name": "variant_hash",
"type": "text",
"primaryKey": false,
"notNull": true
},
"requested_format": {
"name": "requested_format",
"type": "requested_format",
"typeSchema": "public",
"primaryKey": false,
"notNull": true,
"default": "'auto'"
},
"format": {
"name": "format",
"type": "variant_format",
"typeSchema": "public",
"primaryKey": false,
"notNull": true
},
"width": {
"name": "width",
"type": "integer",
"primaryKey": false,
"notNull": true
},
"height": {
"name": "height",
"type": "integer",
"primaryKey": false,
"notNull": false
},
"resize_mode": {
"name": "resize_mode",
"type": "resize_mode",
"typeSchema": "public",
"primaryKey": false,
"notNull": true,
"default": "'fit'"
},
"quality": {
"name": "quality",
"type": "integer",
"primaryKey": false,
"notNull": true
},
"s3_key": {
"name": "s3_key",
"type": "text",
"primaryKey": false,
"notNull": true
},
"content_type": {
"name": "content_type",
"type": "text",
"primaryKey": false,
"notNull": false
},
"etag": {
"name": "etag",
"type": "text",
"primaryKey": false,
"notNull": false
},
"status": {
"name": "status",
"type": "variant_status",
"typeSchema": "public",
"primaryKey": false,
"notNull": true,
"default": "'pending'"
},
"size_bytes": {
"name": "size_bytes",
"type": "bigint",
"primaryKey": false,
"notNull": false
},
"error": {
"name": "error",
"type": "text",
"primaryKey": false,
"notNull": false
},
"attempt_count": {
"name": "attempt_count",
"type": "integer",
"primaryKey": false,
"notNull": true,
"default": 0
},
"last_accessed_at": {
"name": "last_accessed_at",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": false
},
"created_at": {
"name": "created_at",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": true,
"default": "now()"
},
"updated_at": {
"name": "updated_at",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": true,
"default": "now()"
}
},
"indexes": {
"image_variants_lookup_idx": {
"name": "image_variants_lookup_idx",
"columns": [
{
"expression": "asset_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "asset_version",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "preset",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "width",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "height",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "resize_mode",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "quality",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "format",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": true,
"concurrently": false,
"method": "btree",
"with": {}
},
"image_variants_s3_key_idx": {
"name": "image_variants_s3_key_idx",
"columns": [
{
"expression": "s3_key",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": true,
"concurrently": false,
"method": "btree",
"with": {}
},
"image_variants_variant_hash_idx": {
"name": "image_variants_variant_hash_idx",
"columns": [
{
"expression": "variant_hash",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": true,
"concurrently": false,
"method": "btree",
"with": {}
},
"image_variants_status_idx": {
"name": "image_variants_status_idx",
"columns": [
{
"expression": "status",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {
"image_variants_asset_id_image_assets_id_fk": {
"name": "image_variants_asset_id_image_assets_id_fk",
"tableFrom": "image_variants",
"tableTo": "image_assets",
"columnsFrom": [
"asset_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
},
"image_variants_asset_version_id_image_asset_versions_id_fk": {
"name": "image_variants_asset_version_id_image_asset_versions_id_fk",
"tableFrom": "image_variants",
"tableTo": "image_asset_versions",
"columnsFrom": [
"asset_version_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"policies": {},
"checkConstraints": {},
"isRLSEnabled": false
}
},
"enums": {
"public.asset_status": {
"name": "asset_status",
"schema": "public",
"values": [
"active",
"disabled",
"deleted"
]
},
"public.requested_format": {
"name": "requested_format",
"schema": "public",
"values": [
"auto",
"avif",
"webp",
"jpg",
"png"
]
},
"public.resize_mode": {
"name": "resize_mode",
"schema": "public",
"values": [
"fit",
"fill"
]
},
"public.variant_format": {
"name": "variant_format",
"schema": "public",
"values": [
"avif",
"webp",
"jpg",
"png"
]
},
"public.variant_status": {
"name": "variant_status",
"schema": "public",
"values": [
"pending",
"processing",
"ready",
"failed"
]
}
},
"schemas": {},
"sequences": {},
"roles": {},
"policies": {},
"views": {},
"_meta": {
"columns": {},
"schemas": {},
"tables": {}
}
}

View File

@@ -8,6 +8,13 @@
"when": 1777963363578, "when": 1777963363578,
"tag": "0000_calm_magdalene", "tag": "0000_calm_magdalene",
"breakpoints": true "breakpoints": true
},
{
"idx": 1,
"version": "7",
"when": 1777973330318,
"tag": "0001_familiar_nextwave",
"breakpoints": true
} }
] ]
} }

View File

@@ -2,19 +2,20 @@
"name": "@image-platform/database", "name": "@image-platform/database",
"version": "0.1.0", "version": "0.1.0",
"private": true, "private": true,
"type": "module",
"exports": { "exports": {
".": { ".": {
"types": "./src/index.ts", "types": "./src/index.ts",
"require": "./dist/index.js",
"import": "./dist/index.js",
"default": "./dist/index.js" "default": "./dist/index.js"
} }
}, },
"types": "./src/index.ts", "types": "./src/index.ts",
"scripts": { "scripts": {
"build": "tsc -p tsconfig.build.json", "build": "tsc -p tsconfig.build.json",
"db:generate": "drizzle-kit generate --config drizzle.config.ts", "db:generate": "node --env-file-if-exists=../../.env ./node_modules/drizzle-kit/bin.cjs generate --config drizzle.config.ts",
"db:migrate": "drizzle-kit migrate --config drizzle.config.ts", "db:migrate": "node --env-file-if-exists=../../.env ./node_modules/drizzle-kit/bin.cjs migrate --config drizzle.config.ts",
"db:studio": "drizzle-kit studio --config drizzle.config.ts", "db:studio": "node --env-file-if-exists=../../.env ./node_modules/drizzle-kit/bin.cjs studio --config drizzle.config.ts",
"typecheck": "tsc --noEmit -p tsconfig.json" "typecheck": "tsc --noEmit -p tsconfig.json"
}, },
"dependencies": { "dependencies": {

View File

@@ -1,2 +1,3 @@
export { createDatabase, createDatabasePool } from "./client.js" export { createDatabase, createDatabasePool } from "./client.js"
export type { Database, DatabasePool } from "./client.js"
export * from "./schema.js" export * from "./schema.js"

View File

@@ -14,6 +14,7 @@ import {
export const assetStatusEnum = pgEnum("asset_status", ["active", "disabled", "deleted"]) export const assetStatusEnum = pgEnum("asset_status", ["active", "disabled", "deleted"])
export const variantFormatEnum = pgEnum("variant_format", ["avif", "webp", "jpg", "png"]) export const variantFormatEnum = pgEnum("variant_format", ["avif", "webp", "jpg", "png"])
export const requestedFormatEnum = pgEnum("requested_format", ["auto", "avif", "webp", "jpg", "png"]) export const requestedFormatEnum = pgEnum("requested_format", ["auto", "avif", "webp", "jpg", "png"])
export const resizeModeEnum = pgEnum("resize_mode", ["fit", "fill"])
export const variantStatusEnum = pgEnum("variant_status", ["pending", "processing", "ready", "failed"]) export const variantStatusEnum = pgEnum("variant_status", ["pending", "processing", "ready", "failed"])
const timestamps = { const timestamps = {
@@ -86,6 +87,7 @@ export const imageVariants = pgTable(
format: variantFormatEnum("format").notNull(), format: variantFormatEnum("format").notNull(),
width: integer("width").notNull(), width: integer("width").notNull(),
height: integer("height"), height: integer("height"),
resizeMode: resizeModeEnum("resize_mode").notNull().default("fit"),
quality: integer("quality").notNull(), quality: integer("quality").notNull(),
s3Key: text("s3_key").notNull(), s3Key: text("s3_key").notNull(),
contentType: text("content_type"), contentType: text("content_type"),
@@ -103,6 +105,8 @@ export const imageVariants = pgTable(
table.assetVersion, table.assetVersion,
table.preset, table.preset,
table.width, table.width,
table.height,
table.resizeMode,
table.quality, table.quality,
table.format, table.format,
), ),

View File

@@ -5,8 +5,8 @@
"esModuleInterop": true, "esModuleInterop": true,
"forceConsistentCasingInFileNames": true, "forceConsistentCasingInFileNames": true,
"lib": ["ES2023"], "lib": ["ES2023"],
"module": "NodeNext", "module": "Node16",
"moduleResolution": "NodeNext", "moduleResolution": "Node16",
"noUncheckedIndexedAccess": true, "noUncheckedIndexedAccess": true,
"outDir": "./dist", "outDir": "./dist",
"rootDir": "./src", "rootDir": "./src",

View File

@@ -0,0 +1,22 @@
{
"name": "@image-platform/image-config",
"version": "0.1.0",
"private": true,
"exports": {
".": {
"types": "./src/index.ts",
"require": "./dist/index.js",
"import": "./dist/index.js",
"default": "./dist/index.js"
}
},
"types": "./src/index.ts",
"scripts": {
"build": "tsc -p tsconfig.build.json",
"typecheck": "tsc --noEmit -p tsconfig.json"
},
"devDependencies": {
"@types/node": "^25.6.0",
"typescript": "^6.0.3"
}
}

View File

@@ -0,0 +1,20 @@
export const DEFAULT_ALLOWED_SOURCE_HOSTS = ["storage.yandexcloud.net"] as const
export function loadAllowedSourceHostsFromEnv(env: NodeJS.ProcessEnv = process.env) {
const value = env.SOURCE_ALLOWED_HOSTS
if (!value) {
return new Set<string>(DEFAULT_ALLOWED_SOURCE_HOSTS)
}
return new Set(
value
.split(",")
.map((host) => host.trim().toLowerCase())
.filter(Boolean),
)
}
export function isAllowedSourceHost(hostname: string, allowedHosts: ReadonlySet<string>) {
return allowedHosts.has(hostname.toLowerCase())
}

View File

@@ -0,0 +1,2 @@
export * from "./allowed-hosts.js"
export * from "./presets.js"

View File

@@ -0,0 +1,297 @@
export type ActualImageFormat = "avif" | "jpg" | "png" | "webp"
export type RequestedImageFormat = "auto" | ActualImageFormat
export type ResizeMode = "fill" | "fit"
export type PresetMode = "fixed" | "responsive"
export type TransformMode = "custom" | PresetMode
export type ImagePreset = {
formats: readonly ActualImageFormat[]
height?: number
mode: PresetMode
qualities: readonly number[]
quality: number
resize: ResizeMode
width?: number
widths?: readonly number[]
}
export type CustomTransformConfig = {
formats: readonly ActualImageFormat[]
maxHeight: number
maxWidth: number
quality: number
}
export type NormalizeImageTransformInput = {
allowCustomTransforms: boolean
format: ActualImageFormat
height?: number | null
preset: string
quality?: number | null
requestedFormat?: RequestedImageFormat | null
resize?: ResizeMode | null
width?: number | null
}
export type NormalizedImageTransform = {
format: ActualImageFormat
height: number
mode: TransformMode
preset: string
quality: number
requestedFormat: RequestedImageFormat
resize: ResizeMode
width: number
}
export class ImageTransformConfigError extends Error {
constructor(message: string) {
super(message)
this.name = "ImageTransformConfigError"
}
}
export const IMAGE_PRESETS = {
avatar: {
formats: ["avif", "webp", "jpg"],
height: 256,
mode: "fixed",
qualities: [80],
quality: 80,
resize: "fill",
width: 256,
},
card: {
formats: ["avif", "webp", "jpg"],
mode: "responsive",
qualities: [75, 80],
quality: 80,
resize: "fit",
widths: [320, 640, 960],
},
hero: {
formats: ["avif", "webp", "jpg"],
mode: "responsive",
qualities: [75, 80],
quality: 80,
resize: "fit",
widths: [1280, 1920],
},
} as const satisfies Record<string, ImagePreset>
export const CUSTOM_PRESET_NAME = "custom"
export const CUSTOM_TRANSFORM_CONFIG: CustomTransformConfig = {
formats: ["avif", "webp", "jpg", "png"],
maxHeight: 4096,
maxWidth: 4096,
quality: 80,
}
export function getImagePreset(name: string): ImagePreset | null {
return Object.hasOwn(IMAGE_PRESETS, name) ? IMAGE_PRESETS[name as keyof typeof IMAGE_PRESETS] : null
}
export function normalizeImageTransform(input: NormalizeImageTransformInput): NormalizedImageTransform {
const requestedFormat = input.requestedFormat ?? input.format
const preset = getImagePreset(input.preset)
if (!isRequestedImageFormat(requestedFormat)) {
throw new ImageTransformConfigError("requestedFormat is invalid")
}
if (!isActualImageFormat(input.format)) {
throw new ImageTransformConfigError("format is invalid")
}
if (preset) {
return normalizePresetTransform(input, preset, requestedFormat)
}
if (input.preset === CUSTOM_PRESET_NAME) {
return normalizeCustomTransform(input, requestedFormat)
}
throw new ImageTransformConfigError(`unknown image preset: ${input.preset}`)
}
export function selectFormatForAccept(input: {
allowCustomTransforms: boolean
acceptHeader?: string | string[]
preset: string
requestedFormat: string
}): { format: ActualImageFormat; requestedFormat: RequestedImageFormat } {
if (!isRequestedImageFormat(input.requestedFormat)) {
throw new ImageTransformConfigError("requested format is invalid")
}
const formats = getAllowedFormats(input.preset, input.allowCustomTransforms)
if (input.requestedFormat !== "auto") {
if (!formats.includes(input.requestedFormat)) {
throw new ImageTransformConfigError(`format ${input.requestedFormat} is not allowed for ${input.preset}`)
}
return { format: input.requestedFormat, requestedFormat: input.requestedFormat }
}
const accept = Array.isArray(input.acceptHeader) ? input.acceptHeader.join(",") : (input.acceptHeader ?? "")
if (accept.includes("image/avif") && formats.includes("avif")) {
return { format: "avif", requestedFormat: "auto" }
}
if (accept.includes("image/webp") && formats.includes("webp")) {
return { format: "webp", requestedFormat: "auto" }
}
if (formats.includes("jpg")) {
return { format: "jpg", requestedFormat: "auto" }
}
if (formats.includes("png")) {
return { format: "png", requestedFormat: "auto" }
}
throw new ImageTransformConfigError(`no fallback format configured for ${input.preset}`)
}
export function parseBooleanFlag(value: string | undefined, fallback: boolean) {
if (value === undefined) {
return fallback
}
return ["1", "true", "yes"].includes(value.toLowerCase())
}
function normalizePresetTransform(
input: NormalizeImageTransformInput,
preset: ImagePreset,
requestedFormat: RequestedImageFormat,
): NormalizedImageTransform {
if (!preset.formats.includes(input.format)) {
throw new ImageTransformConfigError(`format ${input.format} is not allowed for preset ${input.preset}`)
}
const quality = normalizePresetQuality(input.quality, preset)
if (preset.mode === "fixed") {
if (!preset.width || !preset.height) {
throw new ImageTransformConfigError(`fixed preset ${input.preset} must define width and height`)
}
if (input.width !== null && input.width !== undefined && input.width !== preset.width) {
throw new ImageTransformConfigError(`width must be ${preset.width} for preset ${input.preset}`)
}
if (input.height !== null && input.height !== undefined && input.height !== preset.height) {
throw new ImageTransformConfigError(`height must be ${preset.height} for preset ${input.preset}`)
}
return {
format: input.format,
height: preset.height,
mode: preset.mode,
preset: input.preset,
quality,
requestedFormat,
resize: preset.resize,
width: preset.width,
}
}
if (!input.width) {
throw new ImageTransformConfigError(`width is required for responsive preset ${input.preset}`)
}
if (!preset.widths?.includes(input.width)) {
throw new ImageTransformConfigError(`width ${input.width} is not allowed for preset ${input.preset}`)
}
if (input.height !== null && input.height !== undefined && input.height !== (preset.height ?? 0)) {
throw new ImageTransformConfigError(`height is not configurable for preset ${input.preset}`)
}
return {
format: input.format,
height: preset.height ?? 0,
mode: preset.mode,
preset: input.preset,
quality,
requestedFormat,
resize: preset.resize,
width: input.width,
}
}
function normalizeCustomTransform(
input: NormalizeImageTransformInput,
requestedFormat: RequestedImageFormat,
): NormalizedImageTransform {
if (!input.allowCustomTransforms) {
throw new ImageTransformConfigError("custom transforms are disabled")
}
if (!CUSTOM_TRANSFORM_CONFIG.formats.includes(input.format)) {
throw new ImageTransformConfigError(`format ${input.format} is not allowed for custom transforms`)
}
if (!input.width || input.width > CUSTOM_TRANSFORM_CONFIG.maxWidth) {
throw new ImageTransformConfigError(`custom width must be between 1 and ${CUSTOM_TRANSFORM_CONFIG.maxWidth}`)
}
const height = input.height ?? 0
if (height < 0 || height > CUSTOM_TRANSFORM_CONFIG.maxHeight) {
throw new ImageTransformConfigError(`custom height must be between 0 and ${CUSTOM_TRANSFORM_CONFIG.maxHeight}`)
}
const quality = input.quality ?? CUSTOM_TRANSFORM_CONFIG.quality
if (!Number.isSafeInteger(quality) || quality < 1 || quality > 100) {
throw new ImageTransformConfigError("custom quality must be between 1 and 100")
}
return {
format: input.format,
height,
mode: "custom",
preset: input.preset,
quality,
requestedFormat,
resize: input.resize ?? "fit",
width: input.width,
}
}
function normalizePresetQuality(value: number | null | undefined, preset: ImagePreset) {
const quality = value ?? preset.quality
if (!preset.qualities.includes(quality)) {
throw new ImageTransformConfigError(`quality ${quality} is not allowed for preset`)
}
return quality
}
function getAllowedFormats(presetName: string, allowCustomTransforms: boolean) {
const preset = getImagePreset(presetName)
if (preset) {
return preset.formats
}
if (presetName === CUSTOM_PRESET_NAME && allowCustomTransforms) {
return CUSTOM_TRANSFORM_CONFIG.formats
}
throw new ImageTransformConfigError(`unknown image preset: ${presetName}`)
}
function isActualImageFormat(value: string): value is ActualImageFormat {
return value === "avif" || value === "webp" || value === "jpg" || value === "png"
}
function isRequestedImageFormat(value: string): value is RequestedImageFormat {
return value === "auto" || isActualImageFormat(value)
}

View File

@@ -0,0 +1,7 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"noEmit": false
},
"exclude": ["dist", "node_modules", "**/*.spec.ts"]
}

View File

@@ -0,0 +1,21 @@
{
"compilerOptions": {
"allowSyntheticDefaultImports": true,
"declaration": true,
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"lib": ["ES2023"],
"module": "Node16",
"moduleResolution": "Node16",
"noUncheckedIndexedAccess": true,
"outDir": "./dist",
"rootDir": "./src",
"skipLibCheck": true,
"sourceMap": true,
"strict": true,
"target": "ES2023",
"types": ["node"]
},
"include": ["src/**/*.ts"],
"exclude": ["dist", "node_modules"]
}

View File

@@ -2,10 +2,11 @@
"name": "@image-platform/queue", "name": "@image-platform/queue",
"version": "0.1.0", "version": "0.1.0",
"private": true, "private": true,
"type": "module",
"exports": { "exports": {
".": { ".": {
"types": "./src/index.ts", "types": "./src/index.ts",
"require": "./dist/index.js",
"import": "./dist/index.js",
"default": "./dist/index.js" "default": "./dist/index.js"
} }
}, },
@@ -15,7 +16,11 @@
"typecheck": "tsc --noEmit -p tsconfig.json" "typecheck": "tsc --noEmit -p tsconfig.json"
}, },
"devDependencies": { "devDependencies": {
"@types/amqplib": "^0.10.8",
"@types/node": "^25.6.0", "@types/node": "^25.6.0",
"typescript": "^6.0.3" "typescript": "^6.0.3"
},
"dependencies": {
"amqplib": "^1.0.4"
} }
} }

View File

@@ -0,0 +1,29 @@
import type { Channel } from "amqplib"
import type { GenerateVariantJob } from "./jobs.js"
import type { QueueTopology } from "./topology.js"
export async function assertQueueTopology(channel: Channel, topology: QueueTopology) {
await channel.assertExchange(topology.jobsExchange, "direct", { durable: true })
await channel.assertExchange(topology.jobsDeadLetterExchange, "direct", { durable: true })
await channel.assertQueue(topology.generateVariantQueue, {
deadLetterExchange: topology.jobsDeadLetterExchange,
deadLetterRoutingKey: topology.generateVariantDeadLetterRoutingKey,
durable: true,
})
await channel.assertQueue(topology.generateVariantDeadLetterQueue, { durable: true })
await channel.bindQueue(topology.generateVariantQueue, topology.jobsExchange, topology.generateVariantRoutingKey)
await channel.bindQueue(
topology.generateVariantDeadLetterQueue,
topology.jobsDeadLetterExchange,
topology.generateVariantDeadLetterRoutingKey,
)
}
export function publishGenerateVariantJob(channel: Channel, topology: QueueTopology, job: GenerateVariantJob) {
return channel.publish(topology.jobsExchange, topology.generateVariantRoutingKey, Buffer.from(JSON.stringify(job)), {
contentType: "application/json",
deliveryMode: 2,
persistent: true,
})
}

View File

@@ -1,2 +1,3 @@
export * from "./amqp.js"
export * from "./jobs.js" export * from "./jobs.js"
export * from "./topology.js" export * from "./topology.js"

View File

@@ -5,8 +5,8 @@
"esModuleInterop": true, "esModuleInterop": true,
"forceConsistentCasingInFileNames": true, "forceConsistentCasingInFileNames": true,
"lib": ["ES2023"], "lib": ["ES2023"],
"module": "NodeNext", "module": "Node16",
"moduleResolution": "NodeNext", "moduleResolution": "Node16",
"noUncheckedIndexedAccess": true, "noUncheckedIndexedAccess": true,
"outDir": "./dist", "outDir": "./dist",
"rootDir": "./src", "rootDir": "./src",

View File

@@ -2,10 +2,11 @@
"name": "@image-platform/storage", "name": "@image-platform/storage",
"version": "0.1.0", "version": "0.1.0",
"private": true, "private": true,
"type": "module",
"exports": { "exports": {
".": { ".": {
"types": "./src/index.ts", "types": "./src/index.ts",
"require": "./dist/index.js",
"import": "./dist/index.js",
"default": "./dist/index.js" "default": "./dist/index.js"
} }
}, },

View File

@@ -8,11 +8,15 @@ export type StorageConfig = {
} }
export function loadStorageConfigFromEnv(env: NodeJS.ProcessEnv = process.env): StorageConfig { export function loadStorageConfigFromEnv(env: NodeJS.ProcessEnv = process.env): StorageConfig {
if (!env.S3_BUCKET) {
throw new Error("S3_BUCKET is required")
}
return { return {
accessKeyId: normalizeOptionalString(env.S3_ACCESS_KEY_ID), accessKeyId: normalizeOptionalString(env.S3_ACCESS_KEY_ID),
bucket: env.S3_BUCKET ?? "image-platform", bucket: env.S3_BUCKET,
endpoint: normalizeOptionalString(env.S3_ENDPOINT), endpoint: normalizeOptionalString(env.S3_ENDPOINT),
forcePathStyle: parseBoolean(env.S3_FORCE_PATH_STYLE, true), forcePathStyle: parseBoolean(env.S3_FORCE_PATH_STYLE, false),
region: env.S3_REGION ?? "us-east-1", region: env.S3_REGION ?? "us-east-1",
secretAccessKey: normalizeOptionalString(env.S3_SECRET_ACCESS_KEY), secretAccessKey: normalizeOptionalString(env.S3_SECRET_ACCESS_KEY),
} }

View File

@@ -1,3 +1,4 @@
export * from "./client.js" export * from "./client.js"
export * from "./config.js" export * from "./config.js"
export * from "./keys.js" export * from "./keys.js"
export * from "./objects.js"

View File

@@ -0,0 +1,91 @@
import { GetObjectCommand, HeadObjectCommand, PutObjectCommand, type S3Client } from "@aws-sdk/client-s3"
export type StoredObject = {
body: Buffer
contentLength: number | null
contentType: string | null
etag: string | null
}
export async function getObjectBuffer(client: S3Client, bucket: string, key: string): Promise<StoredObject | null> {
try {
const response = await client.send(new GetObjectCommand({ Bucket: bucket, Key: key }))
if (!response.Body) {
throw new Error(`S3 object ${key} has no body`)
}
const body = await streamToBuffer(response.Body as AsyncIterable<Uint8Array>)
return {
body,
contentLength: response.ContentLength ?? body.length,
contentType: response.ContentType ?? null,
etag: response.ETag ?? null,
}
} catch (error) {
if (isS3NotFound(error)) {
return null
}
throw error
}
}
export async function objectExists(client: S3Client, bucket: string, key: string) {
try {
await client.send(new HeadObjectCommand({ Bucket: bucket, Key: key }))
return true
} catch (error) {
if (isS3NotFound(error)) {
return false
}
throw error
}
}
export async function putObjectBuffer(input: {
body: Buffer
bucket: string
cacheControl?: string
client: S3Client
contentType: string
key: string
}) {
return input.client.send(
new PutObjectCommand({
Body: input.body,
Bucket: input.bucket,
CacheControl: input.cacheControl,
ContentType: input.contentType,
Key: input.key,
}),
)
}
function isS3NotFound(error: unknown) {
if (!(error instanceof Error)) {
return false
}
const withMetadata = error as Error & { $metadata?: { httpStatusCode?: number }; Code?: string; code?: string }
return (
withMetadata.$metadata?.httpStatusCode === 404 ||
error.name === "NoSuchKey" ||
error.name === "NotFound" ||
withMetadata.Code === "NoSuchKey" ||
withMetadata.code === "NoSuchKey"
)
}
async function streamToBuffer(stream: AsyncIterable<Uint8Array>) {
const chunks: Buffer[] = []
for await (const chunk of stream) {
chunks.push(Buffer.from(chunk))
}
return Buffer.concat(chunks)
}

View File

@@ -5,8 +5,8 @@
"esModuleInterop": true, "esModuleInterop": true,
"forceConsistentCasingInFileNames": true, "forceConsistentCasingInFileNames": true,
"lib": ["ES2023"], "lib": ["ES2023"],
"module": "NodeNext", "module": "Node16",
"moduleResolution": "NodeNext", "moduleResolution": "Node16",
"noUncheckedIndexedAccess": true, "noUncheckedIndexedAccess": true,
"outDir": "./dist", "outDir": "./dist",
"rootDir": "./src", "rootDir": "./src",

51
pnpm-lock.yaml generated
View File

@@ -38,6 +38,18 @@ importers:
apps/backend: apps/backend:
dependencies: dependencies:
'@image-platform/database':
specifier: workspace:*
version: link:../../packages/database
'@image-platform/image-config':
specifier: workspace:*
version: link:../../packages/image-config
'@image-platform/queue':
specifier: workspace:*
version: link:../../packages/queue
'@image-platform/storage':
specifier: workspace:*
version: link:../../packages/storage
'@nestjs/common': '@nestjs/common':
specifier: ^11.0.0 specifier: ^11.0.0
version: 11.1.19(reflect-metadata@0.2.2)(rxjs@7.8.2) version: 11.1.19(reflect-metadata@0.2.2)(rxjs@7.8.2)
@@ -50,6 +62,12 @@ importers:
'@nestjs/swagger': '@nestjs/swagger':
specifier: ^11.0.0 specifier: ^11.0.0
version: 11.4.2(@nestjs/common@11.1.19(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.19)(reflect-metadata@0.2.2) version: 11.4.2(@nestjs/common@11.1.19(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.19)(reflect-metadata@0.2.2)
amqplib:
specifier: ^1.0.4
version: 1.0.4
drizzle-orm:
specifier: ^0.45.2
version: 0.45.2(@types/pg@8.20.0)(pg@8.20.0)
reflect-metadata: reflect-metadata:
specifier: ^0.2.2 specifier: ^0.2.2
version: 0.2.2 version: 0.2.2
@@ -66,6 +84,12 @@ importers:
'@nestjs/schematics': '@nestjs/schematics':
specifier: ^11.0.0 specifier: ^11.0.0
version: 11.1.0(chokidar@4.0.3)(typescript@5.9.3) version: 11.1.0(chokidar@4.0.3)(typescript@5.9.3)
'@types/amqplib':
specifier: ^0.10.8
version: 0.10.8
'@types/express':
specifier: ^5.0.6
version: 5.0.6
'@types/node': '@types/node':
specifier: ^24.0.0 specifier: ^24.0.0
version: 24.12.2 version: 24.12.2
@@ -78,6 +102,9 @@ importers:
apps/gateway: apps/gateway:
dependencies: dependencies:
'@image-platform/image-config':
specifier: workspace:*
version: link:../../packages/image-config
fastify: fastify:
specifier: ^5.8.5 specifier: ^5.8.5
version: 5.8.5 version: 5.8.5
@@ -97,6 +124,9 @@ importers:
'@image-platform/database': '@image-platform/database':
specifier: workspace:* specifier: workspace:*
version: link:../../packages/database version: link:../../packages/database
'@image-platform/image-config':
specifier: workspace:*
version: link:../../packages/image-config
'@image-platform/queue': '@image-platform/queue':
specifier: workspace:* specifier: workspace:*
version: link:../../packages/queue version: link:../../packages/queue
@@ -106,6 +136,9 @@ importers:
amqplib: amqplib:
specifier: ^1.0.4 specifier: ^1.0.4
version: 1.0.4 version: 1.0.4
drizzle-orm:
specifier: ^0.45.2
version: 0.45.2(@types/pg@8.20.0)(pg@8.20.0)
devDependencies: devDependencies:
'@types/amqplib': '@types/amqplib':
specifier: ^0.10.8 specifier: ^0.10.8
@@ -142,7 +175,7 @@ importers:
specifier: ^6.0.3 specifier: ^6.0.3
version: 6.0.3 version: 6.0.3
packages/queue: packages/image-config:
devDependencies: devDependencies:
'@types/node': '@types/node':
specifier: ^25.6.0 specifier: ^25.6.0
@@ -151,6 +184,22 @@ importers:
specifier: ^6.0.3 specifier: ^6.0.3
version: 6.0.3 version: 6.0.3
packages/queue:
dependencies:
amqplib:
specifier: ^1.0.4
version: 1.0.4
devDependencies:
'@types/amqplib':
specifier: ^0.10.8
version: 0.10.8
'@types/node':
specifier: ^25.6.0
version: 25.6.0
typescript:
specifier: ^6.0.3
version: 6.0.3
packages/storage: packages/storage:
dependencies: dependencies:
'@aws-sdk/client-s3': '@aws-sdk/client-s3':