ptwm
ptwm
PTWM — lossless compression for PyTorch model weights.
ptwm
python/ptwm/__init__.py:1module ptwmPTWM — lossless compression for PyTorch model weights.
ptwm.CompressionConfig
python/ptwm/_config.py:138class CompressionConfig:ptwm.CompressionConfig.__init__
python/ptwm/_config.py:1def __init__(self, method: Method = Method.AUTO, input_format: Format = Format.BYTE, bytearray_dtype: str = 'bfloat16', is_monotonic: int = 0, threads: int | None = None, compression_threshold: float = 0.95, check_th_after_percent: int = 10, reorder_signbit: int = 0, delta_compressed_type: str | None = None, lossy_compressed_type: Lossy = Lossy.NONE, lossy_compressed_factor: int = 27, compression_chunk: int = 256 * 1024, is_streaming: bool = False, streaming_chunk: int = 1024 * 1024, input_file: str | None = None, compressed_file: str | None = None, decompressed_file: str | None = None, zstd_level: int = 3, lz4_compression_level: int = 0, codec_menu: list[CodecId] | None = None, codec: str | None = None, device: int | None = None, framing: Framing = Framing.COMPACT, container_layout: ContainerLayout = ContainerLayout.COMPACT, emit_payload_hash: bool = True, emit_plane_crc: bool = True) -> Noneptwm.CompressionConfig.bytearray_dtype
python/ptwm/_config.py:142bytearray_dtype: str = 'bfloat16'ptwm.CompressionConfig.check_th_after_percent
python/ptwm/_config.py:146check_th_after_percent: int = 10ptwm.CompressionConfig.codec
python/ptwm/_config.py:167codec: str | None = Noneptwm.CompressionConfig.compressed_file
python/ptwm/_config.py:155compressed_file: str | None = Noneptwm.CompressionConfig.compression_chunk
python/ptwm/_config.py:151compression_chunk: int = 256 * 1024ptwm.CompressionConfig.compression_threshold
python/ptwm/_config.py:145compression_threshold: float = 0.95ptwm.CompressionConfig.container_layout
python/ptwm/_config.py:171container_layout: ContainerLayout = ContainerLayout.COMPACTLayout of a full container written with ``Framing.CONTAINER`` (or as the ``Framing.COMPACT`` fallback): ``COMPACT`` (default, ``PTWC``) or ``LEGACY`` (``PTWM``, readable before v1.2); see :class:`ContainerLayout`. A string such as ``"legacy"`` is coerced on construction.
ptwm.CompressionConfig.decompressed_file
python/ptwm/_config.py:156decompressed_file: str | None = Noneptwm.CompressionConfig.delta_compressed_type
python/ptwm/_config.py:148delta_compressed_type: str | None = Noneptwm.CompressionConfig.device
python/ptwm/_config.py:168device: int | None = Noneptwm.CompressionConfig.emit_payload_hash
python/ptwm/_config.py:177emit_payload_hash: bool = TrueStore an xxh64 of the raw bytes (8 B per tensor), checked on decode.
ptwm.CompressionConfig.emit_plane_crc
python/ptwm/_config.py:179emit_plane_crc: bool = TrueStore a CRC32 of every coded plane (4 B per plane), checked on decode.
ptwm.CompressionConfig.framing
python/ptwm/_config.py:169framing: Framing = Framing.COMPACTOutput layout; see :class:`Framing`.
ptwm.CompressionConfig.from_resolved_policy
python/ptwm/_config.py:199def from_resolved_policy(cls, rp: ResolvedPolicy, base: CompressionConfig | None = None) -> CompressionConfigBuild a constrained ``CompressionConfig`` from a ``ResolvedPolicy``.
ptwm.CompressionConfig.input_file
python/ptwm/_config.py:154input_file: str | None = Noneptwm.CompressionConfig.input_format
python/ptwm/_config.py:141input_format: Format = Format.BYTEptwm.CompressionConfig.is_monotonic
python/ptwm/_config.py:143is_monotonic: int = 0ptwm.CompressionConfig.is_streaming
python/ptwm/_config.py:152is_streaming: bool = Falseptwm.CompressionConfig.lossy_compressed_factor
python/ptwm/_config.py:150lossy_compressed_factor: int = 27ptwm.CompressionConfig.lossy_compressed_type
python/ptwm/_config.py:149lossy_compressed_type: Lossy = Lossy.NONEptwm.CompressionConfig.lz4_compression_level
python/ptwm/_config.py:158lz4_compression_level: int = 0ptwm.CompressionConfig.method
python/ptwm/_config.py:140method: Method = Method.AUTOptwm.CompressionConfig.reorder_signbit
python/ptwm/_config.py:147reorder_signbit: int = 0ptwm.CompressionConfig.streaming_chunk
python/ptwm/_config.py:153streaming_chunk: int = 1024 * 1024ptwm.CompressionConfig.threads
python/ptwm/_config.py:144threads: int | None = Noneptwm.CompressionConfig.zstd_level
python/ptwm/_config.py:157zstd_level: int = 3ptwm.CompressionMethodNotSupportedError
python/ptwm/_exceptions.py:13class CompressionMethodNotSupportedError(Error):Raised when an unsupported compression method is requested.
ptwm.Compressor
python/ptwm/core/_compressor.py:57class Compressor:Stateless compressor for weights data.
ptwm.Compressor.__init__
python/ptwm/core/_compressor.py:60def __init__(self, config: CompressionConfig) -> Noneptwm.Compressor.compress
python/ptwm/core/_compressor.py:64def compress(self, data: bytes | bytearray | np.ndarray | torch.Tensor, delta_second_data: bytes | bytearray | np.ndarray | torch.Tensor | None = None) -> bytesCompress the provided data.
ptwm.Compressor.config
python/ptwm/core/_compressor.py:61config = configptwm.ContainerLayout
python/ptwm/_config.py:81class ContainerLayout(Enum):On-disk layout of a multi-tensor ``.ptwm`` container.
ptwm.ContainerLayout.COMPACT
python/ptwm/_config.py:97COMPACT = 'compact'ptwm.ContainerLayout.LEGACY
python/ptwm/_config.py:98LEGACY = 'legacy'ptwm.DecompressionConfig
python/ptwm/_config.py:260class DecompressionConfig:ptwm.DecompressionConfig.__init__
python/ptwm/_config.py:1def __init__(self, threads: int | None = None, delta_second_data: bytes | None = None, skip_missing: bool = False, codec: str | None = None, device: int | None = None) -> Noneptwm.DecompressionConfig.codec
python/ptwm/_config.py:273codec: str | None = Noneptwm.DecompressionConfig.delta_second_data
python/ptwm/_config.py:265delta_second_data: bytes | None = Noneptwm.DecompressionConfig.device
python/ptwm/_config.py:274device: int | None = Noneptwm.DecompressionConfig.skip_missing
python/ptwm/_config.py:266skip_missing: bool = FalseIf ``True``, open containers that reference non-builtin extensions that are not installed on this host. Tensors needing those extensions will fail when decoded, but the container can still be opened and interrogated (e.g. listing tensor names). When ``False`` (the default), opening such a container raises ``ValueError``.
ptwm.DecompressionConfig.threads
python/ptwm/_config.py:264threads: int | None = Noneptwm.Decompressor
python/ptwm/core/_decompressor.py:31class Decompressor:Stateless decompressor for weights data.
ptwm.Decompressor.__init__
python/ptwm/core/_decompressor.py:34def __init__(self, config: DecompressionConfig | None = None) -> Noneptwm.Decompressor.config
python/ptwm/core/_decompressor.py:35config = config or DecompressionConfig()ptwm.Decompressor.decompress
python/ptwm/core/_decompressor.py:38def decompress(self, data: bytes | memoryview) -> bytes | np.ndarray | torch.TensorDecompress a ``.ptwm`` blob.
ptwm.Decompressor.decompress_container_to_state_dict
python/ptwm/core/_decompressor.py:175def decompress_container_to_state_dict(self, blob: bytes) -> dict[str, bytes]Decompress a container (either layout) into ``{name: raw_bytes}``.
ptwm.Decompressor.decompress_file
python/ptwm/core/_decompressor.py:185def decompress_file(self, path: str) -> bytes | np.ndarray | torch.TensorDecompress data from a file.
ptwm.Error
python/ptwm/_exceptions.py:9class Error(Exception):Base exception for all errors originating from the PTWM library.
ptwm.Format
python/ptwm/_config.py:30class Format(Enum):ptwm.Format.BYTE
python/ptwm/_config.py:31BYTE = 1ptwm.Format.FILE
python/ptwm/_config.py:34FILE = 4ptwm.Format.NUMPY
python/ptwm/_config.py:33NUMPY = 3ptwm.Format.TORCH
python/ptwm/_config.py:32TORCH = 2ptwm.Framing
python/ptwm/_config.py:45class Framing(Enum):Byte layout of a single-tensor :meth:`Compressor.compress` output.
ptwm.Framing.COMPACT
python/ptwm/_config.py:69COMPACT = 1ptwm.Framing.CONTAINER
python/ptwm/_config.py:70CONTAINER = 2ptwm.HeaderParseError
python/ptwm/_exceptions.py:21class HeaderParseError(Error):Raised when the binary header is invalid or corrupt.
ptwm.InvalidDTypeError
python/ptwm/_exceptions.py:17class InvalidDTypeError(Error):Raised when a tensor or numpy array has an unsupported dtype.
ptwm.Lossy
python/ptwm/_config.py:124class Lossy(Enum):ptwm.Lossy.INTEGER
python/ptwm/_config.py:126INTEGER = 1ptwm.Lossy.NONE
python/ptwm/_config.py:125NONE = 0ptwm.Lossy.UNSIGN
python/ptwm/_config.py:127UNSIGN = 2ptwm.materialize_hf_cache
python/ptwm/integrations/_hf.py:137def materialize_hf_cache(snapshot_dir: str | os.PathLike[str], remove_source: bool = False) -> list[Path]Decompress every ``.ptwm`` file under ``snapshot_dir``.
| Name | Type | Description |
|---|---|---|
| snapshot_dir | str | os.PathLike[str] | Any directory; typically an HF cache snapshot (``~/.cache/huggingface/hub/models--<org>--<name>/snapshots/<rev>``). |
| remove_source = False | bool | If ``True``, delete each ``.ptwm`` file once its decompressed sibling is on disk. Defaults to ``False`` (keeps both). |
list of :class:`pathlib.Path` — Paths of the decompressed files, in traversal order.ptwm.Method
python/ptwm/_config.py:13class Method(Enum):ptwm.Method.AUTO
python/ptwm/_config.py:14AUTO = 0ptwm.Method.HUFFMAN
python/ptwm/_config.py:15HUFFMAN = 1ptwm.Method.IDENTITY
python/ptwm/_config.py:17IDENTITY = 3ptwm.Method.MICROSCALE
python/ptwm/_config.py:19MICROSCALE = 5ptwm.Method.RANS
python/ptwm/_config.py:18RANS = 4ptwm.Method.ZSTD
python/ptwm/_config.py:16ZSTD = 2