diff --git a/src/commands/actor/generate-schema-types.ts b/src/commands/actor/generate-schema-types.ts index 6450a0b66..825d39abb 100644 --- a/src/commands/actor/generate-schema-types.ts +++ b/src/commands/actor/generate-schema-types.ts @@ -1,4 +1,4 @@ -import { mkdir, writeFile } from 'node:fs/promises'; +import { mkdir, writeFile, lstat } from 'node:fs/promises'; import path from 'node:path'; import process from 'node:process'; @@ -15,7 +15,7 @@ import { readOutputSchema, readStorageSchema, } from '../../lib/input_schema.js'; -import { error, info, success, warning } from '../../lib/outputs.js'; +import { error, info, simpleLog, success, warning } from '../../lib/outputs.js'; import { clearAllRequired, makePropertiesRequired, @@ -24,6 +24,7 @@ import { prepareOutputSchemaForCompilation, stripTitles, } from '../../lib/schema-transforms.js'; +import { getJsonFileContent, getLocalConfig, getLocalConfigPath, isLocalConfigPath } from '../../lib/utils.js'; export const BANNER_COMMENT = ` // biome-ignore-all lint: generated @@ -85,22 +86,9 @@ Optionally specify custom schema path to use.`; async run() { const cwd = process.cwd(); - - const { inputSchema } = await readAndValidateInputSchema({ - forcePath: this.args.path, - cwd, - getMessage: (schemaPath) => - schemaPath - ? `Generating types from input schema at ${schemaPath}` - : `Generating types from input schema embedded in '${LOCAL_CONFIG_PATH}'`, - }); - - const name = 'input'; - - const schemaToCompile = this.flags.allOptional - ? clearAllRequired(inputSchema) - : makePropertiesRequired(inputSchema); - + const targetSchemas = await this.getTargetSchemas(this.args.path); + await this.prepareOutputPath(cwd, this.flags.output); + const compileOptions: Partial = { bannerComment: BANNER_COMMENT, maxItems: -1, @@ -110,177 +98,164 @@ Optionally specify custom schema path to use.`; $refOptions: { resolve: { external: false, file: false, http: false } }, }; - const result = await compile(stripTitles(schemaToCompile) as JSONSchema4, name, compileOptions); + const [ + inputResult, + datasetResult, + kvsResult, + outputResult, + ] = await Promise.allSettled([ + this.generateInputTypes({schema: targetSchemas.input.inputSchema, compileOptions, allOptional: this.flags.allOptional}), + this.generateDatasetTypes({schema: targetSchemas.dataset?.datasetSchema, compileOptions, allOptional: this.flags.allOptional}), + this.generateKvsTypes({schema: targetSchemas.kvs?.schema, compileOptions, allOptional: this.flags.allOptional}), + this.generateOutputTypes({schema: targetSchemas.output?.schema, compileOptions, allOptional: this.flags.allOptional}), + ]); + } - const outputDir = path.resolve(cwd, this.flags.output); + private async prepareOutputPath(cwd: string, output: string) { + const outputDir = path.resolve(cwd, output); await mkdir(outputDir, { recursive: true }); - - const outputFile = path.join(outputDir, `${name}.ts`); - await writeFile(outputFile, result, 'utf-8'); - - success({ message: `Generated types written to ${outputFile}` }); - - // When no custom path is provided, also generate types from additional schemas - if (!this.args.path) { - const schemaResults = await Promise.allSettled([ - this.generateDatasetTypes({ cwd, outputDir, compileOptions }), - this.generateOutputTypes({ cwd, outputDir, compileOptions }), - this.generateKvsTypes({ cwd, outputDir, compileOptions }), - ]); - - const schemaLabels = ['Dataset', 'Output', 'Key-Value Store']; - let anyFailed = false; - - for (const [i, schemaResult] of schemaResults.entries()) { - if (schemaResult.status === 'rejected') { - anyFailed = true; - error({ - message: `Failed to generate types for ${schemaLabels[i]} schema: ${schemaResult.reason instanceof Error ? schemaResult.reason.message : String(schemaResult.reason)}`, - }); - } - } - - if (anyFailed) { - process.exitCode = CommandExitCodes.BuildFailed; - } - } } - private async generateDatasetTypes({ - cwd, - outputDir, - compileOptions, - }: { - cwd: string; - outputDir: string; - compileOptions: Partial; - }) { - const datasetResult = readDatasetSchema({ cwd }); - - if (!datasetResult) { - return; + /** + * @param targetPath path to target file or directory + * @returns "file" if targetPath is a file, "directory" if targetPath is a directory + * @throws if targetPath is neither a file nor a directory or if targetPath does not exist + */ + private async getPathType(targetPath: string) { + const stats = await lstat(targetPath).catch((err) => { + if (err.code === 'ENOENT') { + throw new Error(`File or directory not found: ${targetPath}`); + } + throw err; + }); + if (stats.isDirectory()) { + return 'directory'; } - - const { datasetSchema, datasetSchemaPath } = datasetResult; - - if (datasetSchemaPath) { - info({ message: `[experimental] Generating types from Dataset schema at ${datasetSchemaPath}` }); - } else { - info({ message: `[experimental] Generating types from Dataset schema embedded in '${LOCAL_CONFIG_PATH}'` }); + if (stats.isFile()) { + return 'file'; } - - const prepared = prepareFieldsSchemaForCompilation(datasetSchema); - - if (!prepared) { - warning({ message: 'Dataset schema has no fields defined, skipping type generation.' }); - return; - } - - const datasetName = 'dataset'; - - const schemaToCompile = this.flags.allOptional ? clearAllRequired(prepared) : prepared; - - const result = await compile(stripTitles(schemaToCompile) as JSONSchema4, datasetName, compileOptions); - - const outputFile = path.join(outputDir, `${datasetName}.ts`); - await writeFile(outputFile, result, 'utf-8'); - - success({ message: `Generated types written to ${outputFile}` }); + throw new Error(`Could not determine type of path: ${targetPath} is neither a file nor a directory`); } - private async generateOutputTypes({ - cwd, - outputDir, - compileOptions, - }: { - cwd: string; - outputDir: string; - compileOptions: Partial; - }) { - const outputResult = readOutputSchema({ cwd }); - - if (!outputResult) { - return; - } - - const { outputSchema, outputSchemaPath } = outputResult; - - if (outputSchemaPath) { - info({ message: `[experimental] Generating types from Output schema at ${outputSchemaPath}` }); - } else { - info({ message: `[experimental] Generating types from Output schema embedded in '${LOCAL_CONFIG_PATH}'` }); - } - - const prepared = prepareOutputSchemaForCompilation(outputSchema); + private notifySchemaFileResolution(schemaName: string, filePath: string | null | undefined) { + return filePath + ? info({ message: `✅ Resolution of ${schemaName} schema found ${filePath}` }) + : info({ message: `❌ Resolution of ${schemaName} schema did not find a definition` }); + } - if (!prepared) { - warning({ message: 'Output schema has no properties defined, skipping type generation.' }); - return; + /** + * @param inputPath path to .actor/actor.json, actor root folder or input schema file + * @throws if inputPath is neither a file nor a directory, inputPath does not exist, or inputPath is not a valid input schema file + */ + private async getTargetSchemas(inputPath = '.') { + const pathType = await this.getPathType(inputPath); + simpleLog({ message: `\n[Schema Resolution]: resolving path of type ${pathType}` }); + + // assume its inputSchema file if file and not localConfig + if (pathType === 'file' && !isLocalConfigPath(inputPath)) { + const inputResult = await readAndValidateInputSchema({ + forcePath: inputPath, + cwd: process.cwd(), + getMessage: (schemaPath) => `✅ Resolution of input schema found ${schemaPath}`, + }); + return { input: inputResult }; } - - const outputName = 'output'; - - const schemaToCompile = this.flags.allOptional ? clearAllRequired(prepared) : prepared; - - const result = await compile(stripTitles(schemaToCompile) as JSONSchema4, outputName, compileOptions); - - const outputFile = path.join(outputDir, `${outputName}.ts`); - await writeFile(outputFile, result, 'utf-8'); - - success({ message: `Generated types written to ${outputFile}` }); + const actorRootPath = + pathType === 'file' + ? inputPath.replace(LOCAL_CONFIG_PATH, '').replace(/\/$/, '') // get rid of /.actor/actor.json + : inputPath; + + const localConfig = pathType === 'file' ? getJsonFileContent(inputPath) : getLocalConfig(inputPath); + + const datasetResult = readDatasetSchema({ cwd: actorRootPath, localConfig }); + const kvsResult = readStorageSchema({ + cwd: actorRootPath, + key: 'keyValueStore', + label: 'Key-Value Store', + localConfig, + }); + const outputResult = readOutputSchema({ cwd: actorRootPath, localConfig }); + const inputResult = await readAndValidateInputSchema({ + cwd: actorRootPath, + localConfig, + getMessage: (schemaPath) => `Validating input schema at ${schemaPath ?? 'NULL'}`, + }); + this.notifySchemaFileResolution('input', inputResult?.inputSchemaPath); + this.notifySchemaFileResolution('dataset', datasetResult?.datasetSchemaPath); + this.notifySchemaFileResolution('kvs', kvsResult?.schemaPath); + this.notifySchemaFileResolution('output', outputResult?.schemaPath); + simpleLog({ message: `[Schema Resolution]: done\n` }); + return { + input: inputResult, + dataset: datasetResult, + kvs: kvsResult, + output: outputResult, + }; } - private async generateKvsTypes({ - cwd, - outputDir, + private async generateInputTypes({ + schema, compileOptions, + allOptional, }: { - cwd: string; - outputDir: string; + schema: Record | null | undefined; compileOptions: Partial; + allOptional: boolean; }) { - const kvsResult = readStorageSchema({ cwd, key: 'keyValueStore', label: 'Key-Value Store' }); + if (!schema) return null; + const preprocessedSchema: JSONSchema4 = stripTitles(allOptional ? clearAllRequired(schema) : schema); + const result = await compile(preprocessedSchema, 'input', compileOptions); + info({ message: '✅ Input schema compiled successfully.' }); + return result; + } - if (!kvsResult) { - return; + private async generateDatasetTypes({ schema, compileOptions, allOptional }: {schema: Record | null | undefined; compileOptions: Partial; allOptional: boolean} ) { + if (!schema) return null; + let preprocessedSchema = prepareFieldsSchemaForCompilation(schema); + if (!preprocessedSchema) return null; + if (allOptional) { + preprocessedSchema = clearAllRequired(preprocessedSchema); } - - const { schema: kvsSchema, schemaPath: kvsSchemaPath } = kvsResult; - - if (kvsSchemaPath) { - info({ message: `[experimental] Generating types from Key-Value Store schema at ${kvsSchemaPath}` }); - } else { - info({ - message: `[experimental] Generating types from Key-Value Store schema embedded in '${LOCAL_CONFIG_PATH}'`, - }); - } - - const collections = prepareKvsCollectionsForCompilation(kvsSchema); - - if (collections.length === 0) { - warning({ - message: 'Key-Value Store schema has no collections with JSON schemas, skipping type generation.', - }); - return; + preprocessedSchema = stripTitles(preprocessedSchema); + const result = await compile(preprocessedSchema, 'dataset', compileOptions); + info({ message: '✅ Dataset schema compiled successfully.' }); + return result; + } + + private async generateOutputTypes({ schema, compileOptions, allOptional }: {schema: Record | null | undefined; compileOptions: Partial; allOptional: boolean} ) { + if (!schema) return null; + let preprocessedSchema = prepareOutputSchemaForCompilation(schema); + if (!preprocessedSchema) return null; + if (allOptional) { + preprocessedSchema = clearAllRequired(preprocessedSchema); } + preprocessedSchema = stripTitles(preprocessedSchema); + const result = await compile(preprocessedSchema, 'output', compileOptions); + info({ message: '✅ Output schema compiled successfully.' }); + return result; + } - const parts: string[] = []; - - for (const { name, schema } of collections) { - const schemaToCompile = this.flags.allOptional ? clearAllRequired(schema) : schema; - - const compiled = await compile(stripTitles(schemaToCompile) as JSONSchema4, name, { - ...compileOptions, - // Only the first collection gets the banner comment - bannerComment: parts.length === 0 ? (compileOptions.bannerComment as string) : '', - }); + private async generateKvsTypes({ schema, compileOptions, allOptional }: { schema: Record | null | undefined; compileOptions: Partial; allOptional: boolean; }) { + if (!schema) return null; + const preparedCollections = prepareKvsCollectionsForCompilation(schema); + if (preparedCollections.length === 0) return null; + + const resultParts = []; + for (const { name, schema: collectionSchema } of preparedCollections) { + let preprocessedSchema = collectionSchema; + if (allOptional) { + preprocessedSchema = clearAllRequired(preprocessedSchema); + } + preprocessedSchema = stripTitles(preprocessedSchema); - parts.push(compiled); + const types = await compile(preprocessedSchema, name, compileOptions); + resultParts.push(types); } - - const outputFile = path.join(outputDir, 'key-value-store.ts'); - await writeFile(outputFile, parts.join('\n'), 'utf-8'); - - success({ message: `Generated types written to ${outputFile}` }); + info({ message: '✅ Key-Value Store schema compiled successfully.' }); + + // should never happen, but just in case + if (resultParts.length === 0) return null; + + return resultParts.join('\n'); } } diff --git a/src/lib/input_schema.ts b/src/lib/input_schema.ts index 9c515c54a..250ad2da2 100644 --- a/src/lib/input_schema.ts +++ b/src/lib/input_schema.ts @@ -24,7 +24,15 @@ const DEFAULT_INPUT_SCHEMA_PATHS = [ * In such a case, path would be set to the location * where the input schema would be expected to be found (and e.g. can be created there). */ -export const readInputSchema = async ({ forcePath, cwd }: { forcePath?: string; cwd: string }) => { +export const readInputSchema = async ({ + forcePath, + cwd, + localConfig, +}: { + forcePath?: string; + cwd: string; + localConfig?: Record; +}) => { if (forcePath) { return { inputSchema: getJsonFileContent(forcePath), @@ -32,7 +40,7 @@ export const readInputSchema = async ({ forcePath, cwd }: { forcePath?: string; }; } - const localConfig = getLocalConfig(cwd); + localConfig ??= getLocalConfig(cwd); if (typeof localConfig?.input === 'object') { return { @@ -80,14 +88,17 @@ export const readAndValidateInputSchema = async ({ forcePath, cwd, getMessage, + localConfig, }: { forcePath?: string; cwd: string; getMessage: (path: string | null) => string; + localConfig?: Record; }): Promise<{ inputSchema: Record; inputSchemaPath: string | null }> => { const { inputSchema, inputSchemaPath } = await readInputSchema({ forcePath, cwd, + localConfig, }); if (!inputSchema) { @@ -115,13 +126,15 @@ export const readStorageSchema = ({ key, label, getRef, + localConfig, }: { cwd: string; key: string; label: string; getRef?: (config: ReturnType) => unknown; + localConfig?: Record; }): { schema: Record; schemaPath: string | null } | null => { - const localConfig = getLocalConfig(cwd); + localConfig ??= getLocalConfig(cwd); const ref = getRef ? getRef(localConfig) : (localConfig?.storages as Record | undefined)?.[key]; @@ -158,10 +171,12 @@ export const readStorageSchema = ({ */ export const readDatasetSchema = ({ cwd, + localConfig, }: { cwd: string; + localConfig?: Record; }): { datasetSchema: Record; datasetSchemaPath: string | null } | null => { - const result = readStorageSchema({ cwd, key: 'dataset', label: 'Dataset' }); + const result = readStorageSchema({ cwd, key: 'dataset', label: 'Dataset', localConfig }); if (!result) { return null; @@ -178,21 +193,8 @@ export const readDatasetSchema = ({ * Thin wrapper around `readStorageSchema` — reads `output` from the top-level config * rather than `storages.`. */ -export const readOutputSchema = ({ - cwd, -}: { - cwd: string; -}): { outputSchema: Record; outputSchemaPath: string | null } | null => { - const result = readStorageSchema({ cwd, key: 'output', label: 'Output', getRef: (config) => config?.output }); - - if (!result) { - return null; - } - - return { - outputSchema: result.schema, - outputSchemaPath: result.schemaPath, - }; +export const readOutputSchema = ({ cwd, localConfig }: { cwd: string; localConfig?: Record }) => { + return readStorageSchema({ cwd, key: 'output', label: 'Output', getRef: (config) => config?.output, localConfig }); }; /** diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 432b1126a..619545f35 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -193,6 +193,7 @@ export async function getLoggedClient(token?: string, apiBaseUrl?: string) { } export const getLocalConfigPath = (cwd: string) => join(cwd, LOCAL_CONFIG_PATH); +export const isLocalConfigPath = (path: string) => path.endsWith(LOCAL_CONFIG_PATH); export const getJsonFileContent = >(filePath: string) => { if (!existsSync(filePath)) { @@ -408,39 +409,41 @@ const parseActorIgnore = async (cwd: string): Promise => { return { excludeFilter, forceIncludePatterns }; }; -/** - * Get Actor local files, omit files defined in .gitignore, .actorignore and .git folder - * All dot files(.file) and folders(.folder/) are included. - */ -export const getActorLocalFilePaths = async (cwd?: string) => { - const resolvedCwd = cwd ?? process.cwd(); - - const hardcodedIgnore = ['.git/**', 'apify_storage', 'node_modules', 'storage', 'crawlee_storage']; - - // Parse .actorignore early to get both exclude filter and force-include patterns - const { excludeFilter: actorignoreFilter, forceIncludePatterns } = await parseActorIgnore(resolvedCwd); - - let gitIgnoreFilter: ((paths: string[]) => string[]) | null = null; - - // Use git ls-files to get gitignored paths — this correctly handles ancestor .gitignore files, - // nested .gitignore files, .git/info/exclude, and global gitignore config +async function getGitIgnoreFilter(cwd: string) { try { const gitIgnored = execSync('git ls-files --others --ignored --exclude-standard --directory', { - cwd: resolvedCwd, + cwd, encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'], }) .split('\n') .filter(Boolean); - if (gitIgnored.length > 0) { - const ig = makeIg().add(gitIgnored); - gitIgnoreFilter = (paths) => paths.filter((p) => !ig.ignores(p)); + if (gitIgnored.length === 0) { + return; } + + const ig = makeIg().add(gitIgnored); + return (paths: string[]) => paths.filter((p) => !ig.ignores(p)); } catch { // git is unavailable or directory is not a git repo — fall back to parsing .gitignore files - gitIgnoreFilter = await getGitignoreFallbackFilter(resolvedCwd); + return await getGitignoreFallbackFilter(cwd); } +} + +/** + * Get Actor local files, omit files defined in .gitignore, .actorignore and .git folder + * All dot files(.file) and folders(.folder/) are included. + */ +export const getActorLocalFilePaths = async (cwd?: string) => { + const resolvedCwd = cwd ?? process.cwd(); + + const hardcodedIgnore = ['.git/**', 'apify_storage', 'node_modules', 'storage', 'crawlee_storage']; + + // Parse .actorignore early to get both exclude filter and force-include patterns + const { excludeFilter: actorignoreFilter, forceIncludePatterns } = await parseActorIgnore(resolvedCwd); + + const gitIgnoreFilter = await getGitIgnoreFilter(resolvedCwd); const allFiles = await glob(['*', '**/**'], { ignore: hardcodedIgnore,