diff --git a/README.md b/README.md index a55cf9e..5b720d3 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,8 @@ Using a buffer account you can split the metadata update into the uploading of t - `--priority-fees `: Priority fees in micro-lamports per compute unit (default: 100000) - `--rpc `: Custom RPC URL - `--export [address]`: Export transactions instead of running them. Optionally specify an override authority address. -- `--export-encoding `: How to encode exported transactions. Choices: none, utf8, base58, base64 (default: base64) +- `--export-encoding `: How to encode exported transactions. Choices: none, utf8, base58, base64, instruction-list (default: base64) +- `--single-extend-per-tx`: Never grow an account by more than 10KB within a single exported transaction. Required when the exported transactions are executed through a CPI, e.g. by a multisig such as Squads. Requires `--export`. - `--tx-version `: Transaction version to build. Choices: legacy, 0 (default: 0) - `-h, --help`: Show help for command @@ -150,7 +151,13 @@ Squads v3 only accepts legacy transactions. If your multisig is on Squads v3, ad npx @solana-program/program-metadata@latest write idl --buffer --export --export-encoding base58 --close-buffer --tx-version legacy ``` -4. Sign the transaction in your multisig and send it +If the metadata account needs to grow by more than 10KB — because it is being created with more than 10KB of data, or updated with more than 10KB of additional data — add `--single-extend-per-tx`. Multisigs execute exported transactions through a CPI and the Solana runtime caps the growth of an account at 10KB per top-level instruction. Therefore, an exported transaction cannot resize an account by more than 10KB. This option spreads the growth over several transactions accordingly, so expect more than one transaction to import: + +```bash +npx @solana-program/program-metadata@latest write idl --buffer --export --export-encoding base58 --close-buffer --single-extend-per-tx +``` + +4. Sign the transaction(s) in your multisig and send them in order ### Examples diff --git a/clients/js/src/cli/commands/create.ts b/clients/js/src/cli/commands/create.ts index eed037c..8d717fb 100644 --- a/clients/js/src/cli/commands/create.ts +++ b/clients/js/src/cli/commands/create.ts @@ -57,6 +57,7 @@ export async function doCreate(seed: Seed, program: Address, file: string | unde programData, seed, metadata, + singleExtendPerTransaction: options.singleExtendPerTx, }), ); } diff --git a/clients/js/src/cli/commands/update-buffer.ts b/clients/js/src/cli/commands/update-buffer.ts index 2fa4eee..4d5941e 100644 --- a/clients/js/src/cli/commands/update-buffer.ts +++ b/clients/js/src/cli/commands/update-buffer.ts @@ -43,6 +43,7 @@ export async function doUpdateBuffer(buffer: Address, file: string | undefined, sourceBuffer: writeInput.buffer, closeSourceBuffer: writeInput.closeBuffer, data: newData, + singleExtendPerTransaction: options.singleExtendPerTx, }), ); } diff --git a/clients/js/src/cli/commands/update.ts b/clients/js/src/cli/commands/update.ts index 102d36c..f46bd67 100644 --- a/clients/js/src/cli/commands/update.ts +++ b/clients/js/src/cli/commands/update.ts @@ -56,6 +56,7 @@ export async function doWrite(seed: Seed, program: Address, file: string | undef program, programData, metadata: metadataAccount, + singleExtendPerTransaction: options.singleExtendPerTx, }), ); } diff --git a/clients/js/src/cli/commands/write.ts b/clients/js/src/cli/commands/write.ts index 5a4109e..fc291bb 100644 --- a/clients/js/src/cli/commands/write.ts +++ b/clients/js/src/cli/commands/write.ts @@ -52,6 +52,7 @@ export async function doWrite(seed: Seed, program: Address, file: string | undef programData, seed, metadata: metadataAccount, + singleExtendPerTransaction: options.singleExtendPerTx, }), ); } diff --git a/clients/js/src/cli/options.ts b/clients/js/src/cli/options.ts index 6a05545..844b803 100644 --- a/clients/js/src/cli/options.ts +++ b/clients/js/src/cli/options.ts @@ -12,6 +12,7 @@ export type GlobalOptions = KeypairOption & RpcOption & ExportOption & ExportEncodingOption & + SingleExtendPerTxOption & TransactionVersionOption; export function setGlobalOptions(command: CustomCommand) { @@ -22,6 +23,7 @@ export function setGlobalOptions(command: CustomCommand) { .addOption(rpcOption) .addOption(exportOption) .addOption(exportEncodingOption) + .addOption(singleExtendPerTxOption) .addOption(transactionVersionOption); } @@ -68,6 +70,15 @@ export const exportEncodingOption = new Option( (value: string): ExportEncoding => (value === 'instruction-list' ? 'instruction-list' : encodingParser(value)), ); +export type SingleExtendPerTxOption = { singleExtendPerTx: boolean }; +export const singleExtendPerTxOption = new Option( + '--single-extend-per-tx', + 'Never grow an account by more than 10KB within a single exported transaction (at most one "extend" instruction per transaction). ' + + 'Required when the exported transactions are executed through a CPI, e.g. by a multisig program such as Squads, ' + + 'where the whole transaction runs as a single top-level instruction and is therefore subject to the 10KB realloc limit ' + + 'that applies per top-level instruction. Requires "--export".', +).default(false); + export type TransactionVersion = 'legacy' | 0; export type TransactionVersionOption = { txVersion: TransactionVersion }; export const transactionVersionOption = new Option( diff --git a/clients/js/src/cli/utils.ts b/clients/js/src/cli/utils.ts index ea3f064..4c8aa6f 100644 --- a/clients/js/src/cli/utils.ts +++ b/clients/js/src/cli/utils.ts @@ -58,6 +58,7 @@ import { NonCanonicalWriteOption, PayerOption, RpcOption, + SingleExtendPerTxOption, WriteOptions, } from './options'; import { createRetryingSolanaRpc, RetryingRpcConfig } from './rpc'; @@ -85,6 +86,7 @@ export class CustomCommand extends Command { export type Client = Awaited>; export async function getClient(options: GlobalOptions) { + assertValidExportOptions(options); const configs = getSolanaConfigs(); const rpcUrl = getRpcUrl(options, configs); const rpcSubscriptionsUrl = getRpcSubscriptionsUrl(rpcUrl, configs); @@ -111,6 +113,18 @@ export async function getClient(options: GlobalOptions) { .use(cliRunOrExport(options)); } +/** + * Rejects option combinations that only make sense when exporting + * transactions. When instructions are executed directly by the CLI, they run + * as top-level instructions in a transaction, so `--single-extend-per-tx` + * would only add needless transactions. + */ +function assertValidExportOptions(options: ExportOption & SingleExtendPerTxOption): void { + if (options.singleExtendPerTx && !options.export) { + logErrorAndExit('The `--single-extend-per-tx` option can only be used together with `--export`.'); + } +} + /** * Shared configuration for the CLI's retrying RPC. Surfaces a warning whenever a * request is rate limited and retried, so a paused command does not appear to diff --git a/clients/js/src/createBuffer.ts b/clients/js/src/createBuffer.ts index dd867f3..a895da3 100644 --- a/clients/js/src/createBuffer.ts +++ b/clients/js/src/createBuffer.ts @@ -20,8 +20,7 @@ import { PROGRAM_METADATA_PROGRAM_ADDRESS, SeedArgs, } from './generated'; -import { REALLOC_LIMIT } from './internals'; -import { getAccountSize, getExtendInstructionPlan, getWriteInstructionPlan } from './utils'; +import { getAccountSize, getExtendInstructionPlan, getWriteInstructionPlan, needsExtend } from './utils'; /** * Builds a plan that creates a brand new buffer account owned by a fresh @@ -120,6 +119,7 @@ export async function getCreateCanonicalBufferInstructionPlan( program: Address; programData: Address; seed: SeedArgs; + singleExtendPerTransaction?: boolean; }, ) { const buffer = input.buffer ?? (await findCanonicalPda({ program: input.program, seed: input.seed }))[0]; @@ -151,6 +151,7 @@ export async function getCreateNonCanonicalBufferInstructionPlan( payer: TransactionSigner; program: Address; seed: SeedArgs; + singleExtendPerTransaction?: boolean; }, ) { const buffer = @@ -177,6 +178,7 @@ async function getPdaBufferInstructionPlan( program: Address; programData?: Address; seed: SeedArgs; + singleExtendPerTransaction?: boolean; }, ) { const dataLength = input.dataLength ?? input.data?.length ?? 0; @@ -194,7 +196,7 @@ async function getPdaBufferInstructionPlan( programData: input.programData, seed: input.seed, }), - ...(dataLength > REALLOC_LIMIT + ...(needsExtend(dataLength) ? [ getExtendInstructionPlan({ account: input.buffer, @@ -202,6 +204,7 @@ async function getPdaBufferInstructionPlan( extraLength: dataLength, program: input.program, programData: input.programData, + singleExtendPerTransaction: input.singleExtendPerTransaction, }), ] : []), diff --git a/clients/js/src/createMetadata.ts b/clients/js/src/createMetadata.ts index f04d117..d250f3d 100644 --- a/clients/js/src/createMetadata.ts +++ b/clients/js/src/createMetadata.ts @@ -25,12 +25,13 @@ import { InitializeInput, PROGRAM_METADATA_PROGRAM_ADDRESS, } from './generated'; -import { isValidInstructionPlan, REALLOC_LIMIT } from './internals'; +import { isValidInstructionPlan } from './internals'; import { getAccountSize, getExtendInstructionPlan, getWriteInstructionPlan, MetadataInput, + needsExtend, resolveMetadataPda, } from './utils'; @@ -66,6 +67,7 @@ export async function getCreateMetadataInstructionPlan( data?: ReadonlyUint8Array; payer: TransactionSigner; closeBuffer?: Address | boolean; + singleExtendPerTransaction?: boolean; }, ): Promise { if (!input.buffer && !input.data) { @@ -115,6 +117,7 @@ export async function getCreateMetadataInstructionPlanUsingNewBuffer( input: Omit & { data: ReadonlyUint8Array; payer: TransactionSigner; + singleExtendPerTransaction?: boolean; }, ) { const rent = await client.getMinimumBalance(Number(getAccountSize(input.data.length))); @@ -131,7 +134,7 @@ export async function getCreateMetadataInstructionPlanUsingNewBuffer( programData: input.programData, seed: input.seed, }), - ...(input.data.length > REALLOC_LIMIT + ...(needsExtend(input.data.length) ? [ getExtendInstructionPlan({ account: input.metadata, @@ -139,6 +142,7 @@ export async function getCreateMetadataInstructionPlanUsingNewBuffer( extraLength: input.data.length, program: input.program, programData: input.programData, + singleExtendPerTransaction: input.singleExtendPerTransaction, }), ] : []), @@ -164,6 +168,7 @@ export async function getCreateMetadataInstructionPlanUsingExistingBuffer( dataLength: number; payer: TransactionSigner; closeBuffer?: Address | boolean; + singleExtendPerTransaction?: boolean; }, ) { const rent = await client.getMinimumBalance(Number(getAccountSize(input.dataLength))); @@ -180,7 +185,7 @@ export async function getCreateMetadataInstructionPlanUsingExistingBuffer( programData: input.programData, seed: input.seed, }), - ...(input.dataLength > REALLOC_LIMIT + ...(needsExtend(input.dataLength) ? [ getExtendInstructionPlan({ account: input.metadata, @@ -188,6 +193,7 @@ export async function getCreateMetadataInstructionPlanUsingExistingBuffer( extraLength: input.dataLength, program: input.program, programData: input.programData, + singleExtendPerTransaction: input.singleExtendPerTransaction, }), ] : []), diff --git a/clients/js/src/internals.ts b/clients/js/src/internals.ts index e2d354f..41a6826 100644 --- a/clients/js/src/internals.ts +++ b/clients/js/src/internals.ts @@ -1,7 +1,5 @@ import { ClientWithTransactionPlanning, InstructionPlan } from '@solana/kit'; -export const REALLOC_LIMIT = 10_240; - /** * Returns `true` if the given instruction plan can be planned by the client's * transaction planner without throwing — i.e. it fits within a single diff --git a/clients/js/src/updateBuffer.ts b/clients/js/src/updateBuffer.ts index adbec3d..6f361b4 100644 --- a/clients/js/src/updateBuffer.ts +++ b/clients/js/src/updateBuffer.ts @@ -11,8 +11,7 @@ import { } from '@solana/kit'; import { Buffer, getCloseInstruction, getTrimInstruction, getWriteInstruction } from './generated'; -import { REALLOC_LIMIT } from './internals'; -import { getExtendInstructionPlan, getWriteInstructionPlan } from './utils'; +import { getExtendInstructionPlan, getWriteInstructionPlan, REALLOC_LIMIT } from './utils'; export async function getUpdateBufferInstructionPlan( client: ClientWithGetMinimumBalance, @@ -24,6 +23,7 @@ export async function getUpdateBufferInstructionPlan( sourceBuffer?: Account; closeSourceBuffer?: Address | boolean; data?: ReadonlyUint8Array; + singleExtendPerTransaction?: boolean; }, ) { if (!input.data && !input.sourceBuffer) { @@ -52,6 +52,7 @@ export async function getUpdateBufferInstructionPlan( account: input.buffer, authority: input.authority, extraLength: Number(input.sizeDifference), + singleExtendPerTransaction: input.singleExtendPerTransaction, }), ] : []), diff --git a/clients/js/src/updateMetadata.ts b/clients/js/src/updateMetadata.ts index 2114994..59557a8 100644 --- a/clients/js/src/updateMetadata.ts +++ b/clients/js/src/updateMetadata.ts @@ -28,8 +28,8 @@ import { Metadata, SetDataInput, } from './generated'; -import { isValidInstructionPlan, REALLOC_LIMIT } from './internals'; -import { getExtendInstructionPlan, MetadataInput, resolveMetadataPda } from './utils'; +import { isValidInstructionPlan } from './internals'; +import { getExtendInstructionPlan, MetadataInput, REALLOC_LIMIT, resolveMetadataPda } from './utils'; type UpdateMetadataClient = ClientWithGetMinimumBalance & ClientWithRpc & @@ -70,6 +70,7 @@ export async function getUpdateMetadataInstructionPlan( data?: ReadonlyUint8Array; payer: TransactionSigner; closeBuffer?: Address | boolean; + singleExtendPerTransaction?: boolean; }, ): Promise { if (!input.buffer && !input.data) { @@ -149,6 +150,7 @@ export async function getUpdateMetadataInstructionPlanUsingNewBuffer( data: ReadonlyUint8Array; metadata: Account; payer: TransactionSigner; + singleExtendPerTransaction?: boolean; }, ) { const sizeDifference = BigInt(input.data.length) - BigInt(input.metadata.data.data.length); @@ -171,6 +173,7 @@ export async function getUpdateMetadataInstructionPlanUsingNewBuffer( extraLength: Number(sizeDifference), program: input.program, programData: input.programData, + singleExtendPerTransaction: input.singleExtendPerTransaction, }), ] : []), @@ -217,6 +220,7 @@ export async function getUpdateMetadataInstructionPlanUsingExistingBuffer( dataLength: number; metadata: Account; payer: TransactionSigner; + singleExtendPerTransaction?: boolean; }, ) { const sizeDifference = BigInt(input.dataLength) - BigInt(input.metadata.data.data.length); @@ -239,6 +243,7 @@ export async function getUpdateMetadataInstructionPlanUsingExistingBuffer( extraLength: Number(sizeDifference), program: input.program, programData: input.programData, + singleExtendPerTransaction: input.singleExtendPerTransaction, }), ] : []), diff --git a/clients/js/src/utils.ts b/clients/js/src/utils.ts index 2b6f33c..b985ff5 100644 --- a/clients/js/src/utils.ts +++ b/clients/js/src/utils.ts @@ -1,5 +1,6 @@ import { Address, + appendTransactionMessageInstruction, assertAccountExists, EncodedAccount, fetchEncodedAccount, @@ -7,16 +8,25 @@ import { getAddressDecoder, getAddressEncoder, getLinearMessagePackerInstructionPlan, + getMessagePackerInstructionPlanFromInstructions, getOptionDecoder, getProgramDerivedAddress, - getReallocMessagePackerInstructionPlan, getStructDecoder, + getTransactionMessageSize, + getTransactionMessageSizeLimit, getU32Decoder, getU64Decoder, + Instruction, MessagePackerInstructionPlan, MicroLamports, ReadonlyUint8Array, Rpc, + SOLANA_ERROR__INSTRUCTION_PLANS__MAX_INSTRUCTIONS_PER_TRANSACTION_EXCEEDED, + SOLANA_ERROR__INSTRUCTION_PLANS__MESSAGE_CANNOT_ACCOMMODATE_PLAN, + SOLANA_ERROR__INSTRUCTION_PLANS__MESSAGE_PACKER_ALREADY_COMPLETE, + SolanaError, + TransactionMessage, + TransactionMessageWithFeePayer, TransactionSigner, unwrapOption, } from '@solana/kit'; @@ -30,11 +40,22 @@ import { FormatArgs, getExtendInstruction, getWriteInstruction, + PROGRAM_METADATA_PROGRAM_ADDRESS, + parseProgramMetadataInstruction, + ProgramMetadataInstruction, SeedArgs, } from './generated'; export const ACCOUNT_HEADER_LENGTH = 96; +/** + * The maximum number of bytes an account can grow by within a single + * instruction, as enforced by the Solana runtime. When instructions are + * executed through a CPI, the limit applies to the whole top-level instruction + * instead. + */ +export const REALLOC_LIMIT = 10_240; + export const LOADER_V3_PROGRAM_ADDRESS = 'BPFLoaderUpgradeab1e11111111111111111111111' as Address<'BPFLoaderUpgradeab1e11111111111111111111111'>; @@ -74,12 +95,38 @@ export type MetadataInput = { * (managed by a third-party authority). */ programData?: Address; + /** + * When `true`, the account is never grown by more than the realloc limit + * (10,240 bytes) within a single transaction, which implies at most one + * `extend` instruction per transaction. + * + * This is required when the resulting transactions are executed through a + * CPI — e.g. by a multisig program such as Squads — since each transaction + * then runs as a single top-level instruction and the whole of it is subject + * to the realloc limit. Defaults to `false`. + */ + singleExtendPerTransaction?: boolean; }; export function getAccountSize(dataLength: bigint | number) { return BigInt(ACCOUNT_HEADER_LENGTH) + BigInt(dataLength); } +/** + * Whether an account created via `allocate` needs explicit `extend` + * instructions to hold `dataLength` bytes of data. + * + * The account grows from nothing to the header length when allocated and to + * `ACCOUNT_HEADER_LENGTH + dataLength` once written, so the header must be + * counted towards the realloc limit. Doing so keeps the creation valid when + * the transactions are executed through a CPI, where the whole transaction + * runs as a single top-level instruction and is therefore subject to the + * limit as a whole. + */ +export function needsExtend(dataLength: number): boolean { + return ACCOUNT_HEADER_LENGTH + dataLength > REALLOC_LIMIT; +} + /** * Resolves the metadata PDA address for the given input. * @@ -184,26 +231,187 @@ function getLoaderV3Decoders() { ] as const; } +/** + * Builds a message packer plan that grows `account` by `extraLength` bytes + * using as many `extend` instructions as needed, each bounded by the + * runtime's realloc limit (10,240 bytes). + * + * By default, extend instructions are packed as densely as transaction size + * allows since the realloc limit applies per top-level instruction. When + * `singleExtendPerTransaction` is `true`, the account is never grown by more + * than the realloc limit within a single transaction — accounting for other + * growing instructions already present in the transaction, such as `allocate` + * — so transactions remain valid when executed through a CPI (e.g. by a + * multisig program). + */ export function getExtendInstructionPlan(input: { account: Address; authority: TransactionSigner; extraLength: number; program?: Address; programData?: Address; + singleExtendPerTransaction?: boolean; }): MessagePackerInstructionPlan { - return getReallocMessagePackerInstructionPlan({ - totalSize: input.extraLength, - getInstruction: size => - getExtendInstruction({ - account: input.account, - authority: input.authority, - length: size, - program: input.program, - programData: input.programData, - }), + const getInstruction = (length: number) => + getExtendInstruction({ + account: input.account, + authority: input.authority, + length, + program: input.program, + programData: input.programData, + }); + + if (input.singleExtendPerTransaction) { + return getSingleExtendMessagePackerInstructionPlan({ + account: input.account, + getInstruction, + totalLength: input.extraLength, + }); + } + + return getMessagePackerInstructionPlanFromInstructions(getReallocChunkSizes(input.extraLength).map(getInstruction)); +} + +/** + * Splits `totalLength` into chunks of at most `REALLOC_LIMIT` bytes. + * + * Stopgap for Kit's `getReallocMessagePackerInstructionPlan`, which emits a + * final 0-byte instruction when `totalLength` is an exact multiple of + * `REALLOC_LIMIT`. Switch back to it once the upstream fix ships. + * + * @example + * ```ts + * getReallocChunkSizes(25_000); // [10240, 10240, 4520] + * getReallocChunkSizes(20_480); // [10240, 10240] + * ``` + */ +function getReallocChunkSizes(totalLength: number): number[] { + const sizes: number[] = []; + for (let remaining = totalLength; remaining > 0; remaining -= REALLOC_LIMIT) { + sizes.push(Math.min(REALLOC_LIMIT, remaining)); + } + return sizes; +} + +/** + * Mirrors the transaction planner's default when no `maxInstructions` + * configuration is provided to the message packer. + */ +const DEFAULT_MAX_INSTRUCTIONS_PER_TRANSACTION = 16; + +/** + * Creates a message packer that grows `account` by `totalLength` bytes whilst + * never growing it by more than `REALLOC_LIMIT` bytes within a single + * transaction message. + * + * Each call packs a single `extend` instruction sized to the growth budget the + * message has left — accounting for program-metadata instructions already + * present in the message, such as `allocate` — or refuses the message when + * that budget is spent so the planner opens a new transaction. + */ +function getSingleExtendMessagePackerInstructionPlan(input: { + account: Address; + getInstruction: (length: number) => Instruction; + totalLength: number; +}): MessagePackerInstructionPlan { + const { account, getInstruction, totalLength } = input; + return Object.freeze({ + getMessagePacker: () => { + let remaining = totalLength; + return Object.freeze({ + done: () => remaining <= 0, + packMessageToCapacity: ( + message: TransactionMessage & TransactionMessageWithFeePayer, + config?: { maxInstructions?: number }, + ) => { + if (remaining <= 0) { + throw new SolanaError(SOLANA_ERROR__INSTRUCTION_PLANS__MESSAGE_PACKER_ALREADY_COMPLETE); + } + + const originalSize = getTransactionMessageSize(message); + const growth = getAccountGrowthInMessage(message, account); + const budget = growth === 'unknown' ? 0 : Math.min(REALLOC_LIMIT - growth, remaining); + if (budget <= 0) { + // The message has no growth budget left for this + // account. Report the size the next instruction would + // need so the planner opens a new message. + const wouldBeSize = getTransactionMessageSize( + appendTransactionMessageInstruction( + getInstruction(Math.min(REALLOC_LIMIT, remaining)), + message, + ), + ); + throw new SolanaError(SOLANA_ERROR__INSTRUCTION_PLANS__MESSAGE_CANNOT_ACCOMMODATE_PLAN, { + numBytesRequired: wouldBeSize - originalSize, + numFreeBytes: getTransactionMessageSizeLimit(message) - originalSize, + }); + } + + const maxInstructions = config?.maxInstructions ?? DEFAULT_MAX_INSTRUCTIONS_PER_TRANSACTION; + if (message.instructions.length >= maxInstructions) { + throw new SolanaError( + SOLANA_ERROR__INSTRUCTION_PLANS__MAX_INSTRUCTIONS_PER_TRANSACTION_EXCEEDED, + { + maxInstructions, + numInstructions: message.instructions.length + 1, + }, + ); + } + + const nextMessage = appendTransactionMessageInstruction(getInstruction(budget), message); + const nextSize = getTransactionMessageSize(nextMessage); + if (nextSize > getTransactionMessageSizeLimit(nextMessage)) { + throw new SolanaError(SOLANA_ERROR__INSTRUCTION_PLANS__MESSAGE_CANNOT_ACCOMMODATE_PLAN, { + numBytesRequired: nextSize - originalSize, + numFreeBytes: getTransactionMessageSizeLimit(nextMessage) - originalSize, + }); + } + + remaining -= budget; + return nextMessage; + }, + }); + }, + kind: 'messagePacker', + planType: 'instructionPlan', }); } +/** + * Estimates by how many bytes `account` is grown by the program-metadata + * instructions already present in the given transaction message. + * + * `allocate` creates the account with the header length and `extend` grows it + * by its explicit length, so both can be accounted for statically. `write` and + * `setData` instead resize the account to an absolute target, so their growth + * depends on the account's size when the transaction lands — on-chain state the + * message cannot tell us — and `'unknown'` is returned. + */ +function getAccountGrowthInMessage(message: TransactionMessage, account: Address): number | 'unknown' { + let growth = 0; + for (const instruction of message.instructions) { + if (instruction.programAddress !== PROGRAM_METADATA_PROGRAM_ADDRESS || !instruction.data) continue; + const parsed = parseProgramMetadataInstruction({ ...instruction, data: instruction.data }); + switch (parsed.instructionType) { + case ProgramMetadataInstruction.Allocate: + if (parsed.accounts.buffer.address === account) growth += ACCOUNT_HEADER_LENGTH; + break; + case ProgramMetadataInstruction.Extend: + if (parsed.accounts.account.address === account) growth += parsed.data.length; + break; + case ProgramMetadataInstruction.Write: + if (parsed.accounts.buffer.address === account) return 'unknown'; + break; + case ProgramMetadataInstruction.SetData: + if (parsed.accounts.metadata.address === account) return 'unknown'; + break; + default: + break; + } + } + return growth; +} + export function getWriteInstructionPlan(input: { buffer: Address; authority: TransactionSigner; diff --git a/clients/js/src/writeMetadata.ts b/clients/js/src/writeMetadata.ts index 178b358..22d6f5a 100644 --- a/clients/js/src/writeMetadata.ts +++ b/clients/js/src/writeMetadata.ts @@ -56,6 +56,7 @@ export async function getWriteMetadataInstructionPlan( data?: ReadonlyUint8Array; metadata: MaybeAccount; payer: TransactionSigner; + singleExtendPerTransaction?: boolean; }, ): Promise { return input.metadata.exists diff --git a/clients/js/test/_setup.ts b/clients/js/test/_setup.ts index c5280e5..763c080 100644 --- a/clients/js/test/_setup.ts +++ b/clients/js/test/_setup.ts @@ -29,7 +29,7 @@ import { programMetadataProgram, } from '../src'; -export const REALLOC_LIMIT = 10_240; +export { REALLOC_LIMIT } from '../src'; const PROGRAM_METADATA_BINARY_PATH = path.resolve( __dirname, diff --git a/clients/js/test/createMetadata.test.ts b/clients/js/test/createMetadata.test.ts index 8b7f8b0..93e2449 100644 --- a/clients/js/test/createMetadata.test.ts +++ b/clients/js/test/createMetadata.test.ts @@ -11,7 +11,7 @@ import { Format, Metadata, } from '../src'; -import { createDeployedProgram, createTestClient, generateKeyPairSignerWithSol } from './_setup'; +import { createDeployedProgram, createTestClient, generateKeyPairSignerWithSol, REALLOC_LIMIT } from './_setup'; it('creates a canonical metadata account', async () => { // Given the following authority and deployed program. @@ -91,6 +91,52 @@ it('creates a canonical metadata account with data larger than a transaction siz }); }); +it.each([ + { singleExtendPerTransaction: false, label: 'densely packed extend instructions' }, + { singleExtendPerTransaction: true, label: 'a single extend instruction per transaction' }, +])( + 'creates a canonical metadata account with data larger than the realloc limit using $label', + async ({ singleExtendPerTransaction }) => { + // Given the following authority and deployed program. + const client = await createTestClient(); + const authority = await generateKeyPairSignerWithSol(client); + const [program, programData] = await createDeployedProgram(client, authority); + + // When we create a canonical metadata account with more data than the realloc limit. + const largeData = getUtf8Encoder().encode('x'.repeat(2 * REALLOC_LIMIT + 4_520)); + await client.programMetadata.createMetadata({ + authority, + program, + programData, + seed: 'idl', + encoding: Encoding.Utf8, + compression: Compression.None, + dataSource: DataSource.Direct, + format: Format.Json, + data: largeData, + singleExtendPerTransaction, + }); + + // Then we expect the following metadata account to be created. + const [metadata] = await findCanonicalPda({ program, seed: 'idl' }); + const account = await client.programMetadata.accounts.metadata.fetch(metadata); + expect(account.data).toMatchObject({ + discriminator: AccountDiscriminator.Metadata, + program, + authority: none(), + mutable: true, + canonical: true, + seed: 'idl', + encoding: Encoding.Utf8, + compression: Compression.None, + format: Format.Json, + dataSource: DataSource.Direct, + dataLength: largeData.length, + data: largeData, + }); + }, +); + it('creates a canonical metadata account using an existing buffer', async () => { // Given the following authority and deployed program. const client = await createTestClient(); @@ -136,6 +182,58 @@ it('creates a canonical metadata account using an existing buffer', async () => }); }); +it.each([ + { singleExtendPerTransaction: false, label: 'densely packed extend instructions' }, + { singleExtendPerTransaction: true, label: 'a single extend instruction per transaction' }, +])( + 'creates a canonical metadata account using an existing buffer as large as the realloc limit using $label', + async ({ singleExtendPerTransaction }) => { + // Given the following authority and deployed program. + const client = await createTestClient(); + const authority = await generateKeyPairSignerWithSol(client); + const [program, programData] = await createDeployedProgram(client, authority); + + // And an existing buffer holding exactly one realloc limit of data. + const data = getUtf8Encoder().encode('x'.repeat(REALLOC_LIMIT)); + const buffer = await generateKeyPairSigner(); + await client.programMetadata.instructions + .createBuffer({ newBuffer: buffer, authority: buffer, data }) + .sendTransactions(); + + // When we create a canonical metadata account using the existing buffer. + await client.programMetadata.createMetadata({ + authority, + program, + programData, + seed: 'idl', + encoding: Encoding.Utf8, + compression: Compression.None, + dataSource: DataSource.Direct, + format: Format.Json, + buffer: buffer.address, + singleExtendPerTransaction, + }); + + // Then we expect the following metadata account to be created. + const [metadata] = await findCanonicalPda({ program, seed: 'idl' }); + const account = await client.programMetadata.accounts.metadata.fetch(metadata); + expect(account.data).toMatchObject({ + discriminator: AccountDiscriminator.Metadata, + program, + authority: none(), + mutable: true, + canonical: true, + seed: 'idl', + encoding: Encoding.Utf8, + compression: Compression.None, + format: Format.Json, + dataSource: DataSource.Direct, + dataLength: data.length, + data, + }); + }, +); + it('creates a non-canonical metadata account', async () => { // Given the following authority and deployed program. const client = await createTestClient(); diff --git a/clients/js/test/extendInstructionPlan.test.ts b/clients/js/test/extendInstructionPlan.test.ts new file mode 100644 index 0000000..ea30e59 --- /dev/null +++ b/clients/js/test/extendInstructionPlan.test.ts @@ -0,0 +1,291 @@ +import { SYSTEM_PROGRAM_ADDRESS } from '@solana-program/system'; +import { + Address, + flattenTransactionPlan, + generateKeyPairSigner, + getUtf8Encoder, + InstructionPlan, + TransactionMessage, +} from '@solana/kit'; +import { expect, it } from 'vitest'; + +import { + ACCOUNT_HEADER_LENGTH, + Compression, + DataSource, + Encoding, + findCanonicalPda, + Format, + getCreateMetadataInstructionPlanUsingExistingBuffer, + getCreateMetadataInstructionPlanUsingNewBuffer, + getExtendInstructionPlan, + parseProgramMetadataInstruction, + PROGRAM_METADATA_PROGRAM_ADDRESS, + ProgramMetadataInstruction, +} from '../src'; +import { + createDeployedProgram, + createTestClient, + generateKeyPairSignerWithSol, + REALLOC_LIMIT, + TestClient, +} from './_setup'; + +it('packs several extend instructions per transaction by default', async () => { + // Given an extend plan growing an account by more than two realloc limits. + const client = await createTestClient(); + const authority = await generateKeyPairSignerWithSol(client); + const account = (await findCanonicalPda({ program: authority.address, seed: 'idl' }))[0]; + const plan = getExtendInstructionPlan({ account, authority, extraLength: 25_000 }); + + // When we plan it into transactions. + const messages = await planMessages(client, plan); + + // Then all extend instructions fit into a single transaction, each bounded by the realloc limit. + expect(messages).toHaveLength(1); + expect(getExtendLengths(messages[0], account)).toEqual([REALLOC_LIMIT, REALLOC_LIMIT, 25_000 - 2 * REALLOC_LIMIT]); +}); + +it('covers the full length when it is an exact multiple of the realloc limit', async () => { + // Given an extend plan whose length is exactly two realloc limits. + const client = await createTestClient(); + const authority = await generateKeyPairSignerWithSol(client); + const account = (await findCanonicalPda({ program: authority.address, seed: 'idl' }))[0]; + const plan = getExtendInstructionPlan({ account, authority, extraLength: 2 * REALLOC_LIMIT }); + + // When we plan it into transactions. + const messages = await planMessages(client, plan); + + // Then we expect exactly two full extend instructions and no empty one. + expect(messages).toHaveLength(1); + expect(getExtendLengths(messages[0], account)).toEqual([REALLOC_LIMIT, REALLOC_LIMIT]); +}); + +it('packs at most one extend instruction per transaction when requested', async () => { + // Given an extend plan growing an account by more than two realloc limits, + // constrained to a single extend instruction per transaction. + const client = await createTestClient(); + const authority = await generateKeyPairSignerWithSol(client); + const account = (await findCanonicalPda({ program: authority.address, seed: 'idl' }))[0]; + const plan = getExtendInstructionPlan({ + account, + authority, + extraLength: 25_000, + singleExtendPerTransaction: true, + }); + + // When we plan it into transactions. + const messages = await planMessages(client, plan); + + // Then each transaction holds exactly one extend instruction. + expect(messages.map(message => getExtendLengths(message, account))).toEqual([ + [REALLOC_LIMIT], + [REALLOC_LIMIT], + [25_000 - 2 * REALLOC_LIMIT], + ]); +}); + +it('never grows an account by more than the realloc limit per transaction when creating metadata', async () => { + // Given a deployed program and a metadata payload larger than two realloc limits. + const client = await createTestClient(); + const authority = await generateKeyPairSignerWithSol(client); + const [program, programData] = await createDeployedProgram(client, authority); + const [metadata] = await findCanonicalPda({ program, seed: 'idl' }); + const data = getUtf8Encoder().encode('x'.repeat(25_000)); + + // When we plan its creation with a single extend instruction per transaction. + const plan = await getCreateMetadataInstructionPlanUsingNewBuffer(client, { + authority, + data, + metadata, + payer: authority, + program, + programData, + seed: 'idl', + encoding: Encoding.Utf8, + compression: Compression.None, + dataSource: DataSource.Direct, + format: Format.Json, + singleExtendPerTransaction: true, + }); + const messages = await planMessages(client, plan); + + // Then the first extend shares its transaction with the rent transfer and the + // allocation, and is shrunk by the account header the allocation creates. + expect(hasInstructionFrom(messages[0], SYSTEM_PROGRAM_ADDRESS)).toBe(true); + expect(getInstructionTypes(messages[0], metadata)).toEqual([ + ProgramMetadataInstruction.Allocate, + ProgramMetadataInstruction.Extend, + ]); + expect(getExtendLengths(messages[0], metadata)).toEqual([REALLOC_LIMIT - ACCOUNT_HEADER_LENGTH]); + + // And the remaining extend instructions each get their own transaction, the + // last one being followed by the write instructions. + expect(getInstructionTypes(messages[1], metadata)).toEqual([ProgramMetadataInstruction.Extend]); + expect(getExtendLengths(messages[1], metadata)).toEqual([REALLOC_LIMIT]); + expect(getInstructionTypes(messages[2], metadata).slice(0, 2)).toEqual([ + ProgramMetadataInstruction.Extend, + ProgramMetadataInstruction.Write, + ]); + + // And the account never grows by more than the realloc limit within a transaction + // whilst the extend instructions add up to the full data length. + const growths = messages.map(message => getGrowth(message, metadata)); + expect(growths.every(growth => growth <= REALLOC_LIMIT)).toBe(true); + expect(messages.flatMap(message => getExtendLengths(message, metadata)).reduce((a, b) => a + b, 0)).toBe( + data.length, + ); +}); + +it('packs the extend instructions densely when creating metadata by default', async () => { + // Given a deployed program and a metadata payload larger than two realloc limits. + const client = await createTestClient(); + const authority = await generateKeyPairSignerWithSol(client); + const [program, programData] = await createDeployedProgram(client, authority); + const [metadata] = await findCanonicalPda({ program, seed: 'idl' }); + const data = getUtf8Encoder().encode('x'.repeat(25_000)); + + // When we plan its creation without constraining the extend instructions. + const plan = await getCreateMetadataInstructionPlanUsingNewBuffer(client, { + authority, + data, + metadata, + payer: authority, + program, + programData, + seed: 'idl', + encoding: Encoding.Utf8, + compression: Compression.None, + dataSource: DataSource.Direct, + format: Format.Json, + }); + const messages = await planMessages(client, plan); + + // Then all extend instructions share the first transaction with the allocation. + expect(getInstructionTypes(messages[0], metadata).slice(0, 4)).toEqual([ + ProgramMetadataInstruction.Allocate, + ProgramMetadataInstruction.Extend, + ProgramMetadataInstruction.Extend, + ProgramMetadataInstruction.Extend, + ]); + expect(getExtendLengths(messages[0], metadata)).toEqual([REALLOC_LIMIT, REALLOC_LIMIT, 25_000 - 2 * REALLOC_LIMIT]); +}); + +it('accounts for the header when the data alone fits within the realloc limit', async () => { + // Given a deployed program and an existing buffer holding exactly one realloc limit of data, + // which together with the account header exceeds the limit. + const client = await createTestClient(); + const authority = await generateKeyPairSignerWithSol(client); + const [program, programData] = await createDeployedProgram(client, authority); + const [metadata] = await findCanonicalPda({ program, seed: 'idl' }); + const buffer = await generateKeyPairSigner(); + + // When we plan the metadata creation from that buffer with a single extend instruction per transaction. + const plan = await getCreateMetadataInstructionPlanUsingExistingBuffer(client, { + authority, + buffer: buffer.address, + dataLength: REALLOC_LIMIT, + metadata, + payer: authority, + program, + programData, + seed: 'idl', + encoding: Encoding.Utf8, + compression: Compression.None, + dataSource: DataSource.Direct, + format: Format.Json, + singleExtendPerTransaction: true, + }); + const messages = await planMessages(client, plan); + + // Then the allocation and a header-adjusted extend share the first transaction, + // and the remaining bytes are extended before the write in the second one. + expect(getInstructionTypes(messages[0], metadata)).toEqual([ + ProgramMetadataInstruction.Allocate, + ProgramMetadataInstruction.Extend, + ]); + expect(getExtendLengths(messages[0], metadata)).toEqual([REALLOC_LIMIT - ACCOUNT_HEADER_LENGTH]); + expect(getInstructionTypes(messages[1], metadata)).toEqual([ + ProgramMetadataInstruction.Extend, + ProgramMetadataInstruction.Write, + ProgramMetadataInstruction.Initialize, + ]); + expect(getExtendLengths(messages[1], metadata)).toEqual([ACCOUNT_HEADER_LENGTH]); + expect(messages.every(message => getGrowth(message, metadata) <= REALLOC_LIMIT)).toBe(true); +}); + +it('adds a single extend instruction when the data alone fits within the realloc limit by default', async () => { + // Given a deployed program and an existing buffer holding exactly one realloc limit of data. + const client = await createTestClient(); + const authority = await generateKeyPairSignerWithSol(client); + const [program, programData] = await createDeployedProgram(client, authority); + const [metadata] = await findCanonicalPda({ program, seed: 'idl' }); + const buffer = await generateKeyPairSigner(); + + // When we plan the metadata creation from that buffer without constraining the extend instructions. + const plan = await getCreateMetadataInstructionPlanUsingExistingBuffer(client, { + authority, + buffer: buffer.address, + dataLength: REALLOC_LIMIT, + metadata, + payer: authority, + program, + programData, + seed: 'idl', + encoding: Encoding.Utf8, + compression: Compression.None, + dataSource: DataSource.Direct, + format: Format.Json, + }); + const messages = await planMessages(client, plan); + + // Then everything fits in a single transaction with one full extend instruction. + expect(messages).toHaveLength(1); + expect(getInstructionTypes(messages[0], metadata)).toEqual([ + ProgramMetadataInstruction.Allocate, + ProgramMetadataInstruction.Extend, + ProgramMetadataInstruction.Write, + ProgramMetadataInstruction.Initialize, + ]); + expect(getExtendLengths(messages[0], metadata)).toEqual([REALLOC_LIMIT]); +}); + +async function planMessages(client: TestClient, plan: InstructionPlan): Promise { + const transactionPlan = await client.planTransactions(plan); + return flattenTransactionPlan(transactionPlan).map(single => single.message); +} + +function getProgramMetadataInstructions(message: TransactionMessage, account: Address) { + return message.instructions + .filter(instruction => instruction.programAddress === PROGRAM_METADATA_PROGRAM_ADDRESS && instruction.data) + .map(instruction => parseProgramMetadataInstruction({ ...instruction, data: instruction.data! })) + .filter(parsed => Object.values(parsed.accounts)[0]?.address === account); +} + +function getInstructionTypes(message: TransactionMessage, account: Address): ProgramMetadataInstruction[] { + return getProgramMetadataInstructions(message, account).map(parsed => parsed.instructionType); +} + +function getExtendLengths(message: TransactionMessage, account: Address): number[] { + return getProgramMetadataInstructions(message, account).flatMap(parsed => + parsed.instructionType === ProgramMetadataInstruction.Extend ? [parsed.data.length] : [], + ); +} + +/** Mirrors the runtime's per-transaction accounting of an account's data growth. */ +function getGrowth(message: TransactionMessage, account: Address): number { + return getProgramMetadataInstructions(message, account).reduce((growth, parsed) => { + switch (parsed.instructionType) { + case ProgramMetadataInstruction.Allocate: + return growth + ACCOUNT_HEADER_LENGTH; + case ProgramMetadataInstruction.Extend: + return growth + parsed.data.length; + default: + return growth; + } + }, 0); +} + +function hasInstructionFrom(message: TransactionMessage, programAddress: Address): boolean { + return message.instructions.some(instruction => instruction.programAddress === programAddress); +} diff --git a/clients/js/test/updateMetadata.test.ts b/clients/js/test/updateMetadata.test.ts index 2d52331..84ac39d 100644 --- a/clients/js/test/updateMetadata.test.ts +++ b/clients/js/test/updateMetadata.test.ts @@ -11,7 +11,7 @@ import { Format, Metadata, } from '../src'; -import { createDeployedProgram, createTestClient, generateKeyPairSignerWithSol } from './_setup'; +import { createDeployedProgram, createTestClient, generateKeyPairSignerWithSol, REALLOC_LIMIT } from './_setup'; it('updates a canonical metadata account', async () => { // Given the following authority and deployed program. @@ -175,6 +175,71 @@ it('updates a canonical metadata account using an existing buffer', async () => }); }); +it.each([ + { singleExtendPerTransaction: false, label: 'densely packed extend instructions' }, + { singleExtendPerTransaction: true, label: 'a single extend instruction per transaction' }, +])( + 'updates a canonical metadata account using an existing buffer larger than the realloc limit using $label', + async ({ singleExtendPerTransaction }) => { + // Given the following authority and deployed program. + const client = await createTestClient(); + const authority = await generateKeyPairSignerWithSol(client); + const [program, programData] = await createDeployedProgram(client, authority); + + // And the following existing canonical metadata account. + await client.programMetadata.createMetadata({ + authority, + program, + programData, + seed: 'idl', + encoding: Encoding.Utf8, + compression: Compression.None, + dataSource: DataSource.Direct, + format: Format.Json, + data: getUtf8Encoder().encode('OLD'), + }); + + // And an existing buffer holding more data than the realloc limit. + const newData = getUtf8Encoder().encode('x'.repeat(2 * REALLOC_LIMIT + 4_520)); + const buffer = await generateKeyPairSigner(); + await client.programMetadata.instructions + .createBuffer({ newBuffer: buffer, authority: buffer, data: newData }) + .sendTransactions(); + + // When we update the metadata account using the existing buffer. + await client.programMetadata.updateMetadata({ + authority, + program, + programData, + seed: 'idl', + encoding: Encoding.Base58, + compression: Compression.Gzip, + dataSource: DataSource.Url, + format: Format.Toml, + buffer: buffer.address, + singleExtendPerTransaction, + }); + + // Then we expect the metadata account to be updated. + const [metadata] = await findCanonicalPda({ program, seed: 'idl' }); + const account = await client.programMetadata.accounts.metadata.fetch(metadata); + expect(account.data).toMatchObject({ + discriminator: AccountDiscriminator.Metadata, + program, + authority: none(), + mutable: true, + canonical: true, + seed: 'idl', + encoding: Encoding.Base58, + compression: Compression.Gzip, + dataSource: DataSource.Url, + format: Format.Toml, + dataLength: newData.length, + data: newData, + }); + }, +); + it('updates a non-canonical metadata account', async () => { // Given the following authority and deployed program. const client = await createTestClient();