multipart

s3_asyncio_client.multipart

Asyncio-based multipart upload functionality for S3.

Based on s3transfer library architecture but implemented with asyncio. Uses functions and coroutines instead of complex class hierarchies and threads.

adjust_chunk_size(current_chunksize, file_size=None)

Adjust chunk size to comply with S3 limits. Based on s3transfer's ChunksizeAdjuster logic.

Source code in src/s3_asyncio_client/multipart.py

def adjust_chunk_size(current_chunksize: int, file_size: int | None = None) -> int:
    """Adjust chunk size to comply with S3 limits.
    Based on s3transfer's ChunksizeAdjuster logic.
    """
    chunksize = current_chunksize

    if file_size is not None:
        chunksize = _adjust_for_max_parts(chunksize, file_size)

    return _adjust_for_size_limits(chunksize)

read_file_chunks(file_path, part_size) async

Async generator that yields file chunks for multipart upload.

Source code in src/s3_asyncio_client/multipart.py

async def read_file_chunks(
    file_path: str | Path, part_size: int
) -> AsyncGenerator[bytes, None]:
    """Async generator that yields file chunks for multipart upload."""
    file_path = Path(file_path)

    def read_chunk():
        with open(file_path, "rb") as f:
            while True:
                chunk = f.read(part_size)
                if not chunk:
                    break
                yield chunk

    # TODO: Use asyncio to make file reading non-blocking

    for chunk in read_chunk():
        await asyncio.sleep(0)
        yield chunk

read_fileobj_chunks(fileobj, part_size) async

Async generator that yields chunks from a file-like object.

Source code in src/s3_asyncio_client/multipart.py

async def read_fileobj_chunks(fileobj, part_size: int) -> AsyncGenerator[bytes, None]:
    """Async generator that yields chunks from a file-like object."""
    while True:
        # Use asyncio to make reading non-blocking
        loop = asyncio.get_event_loop()
        chunk = await loop.run_in_executor(None, fileobj.read, part_size)

        if not chunk:
            break

        yield chunk